2021-04-19 15:16:33 +08:00
|
|
|
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
|
|
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
|
|
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
|
|
|
2021-05-08 15:24:12 +08:00
|
|
|
// Package datanode implements data persistence logic.
|
|
|
|
//
|
2021-06-11 17:53:37 +08:00
|
|
|
// Data node persists insert logs into persistent storage like minIO/S3.
|
2021-01-19 11:37:16 +08:00
|
|
|
package datanode
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2021-03-23 18:50:13 +08:00
|
|
|
"errors"
|
2021-02-03 17:30:10 +08:00
|
|
|
"fmt"
|
2021-01-19 11:37:16 +08:00
|
|
|
"io"
|
2021-03-08 15:25:55 +08:00
|
|
|
"math/rand"
|
2021-05-21 19:28:52 +08:00
|
|
|
"strconv"
|
2021-08-13 10:50:09 +08:00
|
|
|
"strings"
|
2021-05-25 15:35:37 +08:00
|
|
|
"sync"
|
2021-02-04 20:31:23 +08:00
|
|
|
"sync/atomic"
|
2021-01-24 21:20:11 +08:00
|
|
|
"time"
|
2021-01-19 11:37:16 +08:00
|
|
|
|
2021-08-24 09:45:51 +08:00
|
|
|
clientv3 "go.etcd.io/etcd/client/v3"
|
2021-02-26 10:13:36 +08:00
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2021-08-13 10:50:09 +08:00
|
|
|
"github.com/golang/protobuf/proto"
|
|
|
|
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/log"
|
2021-08-13 10:50:09 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/logutil"
|
2021-06-21 18:08:15 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/metrics"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/msgstream"
|
|
|
|
"github.com/milvus-io/milvus/internal/types"
|
2021-09-01 10:13:15 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/metricsinfo"
|
2021-08-13 10:50:09 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/retry"
|
2021-05-21 19:28:52 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/sessionutil"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/typeutil"
|
2021-03-05 20:41:34 +08:00
|
|
|
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/commonpb"
|
|
|
|
"github.com/milvus-io/milvus/internal/proto/datapb"
|
|
|
|
"github.com/milvus-io/milvus/internal/proto/internalpb"
|
|
|
|
"github.com/milvus-io/milvus/internal/proto/milvuspb"
|
2021-06-22 16:14:09 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
|
2021-01-19 11:37:16 +08:00
|
|
|
)
|
|
|
|
|
2021-01-24 21:20:11 +08:00
|
|
|
const (
|
|
|
|
RPCConnectionTimeout = 30 * time.Second
|
2021-06-21 18:08:15 +08:00
|
|
|
|
|
|
|
// MetricRequestsTotal used to count the num of total requests
|
|
|
|
MetricRequestsTotal = "total"
|
|
|
|
|
|
|
|
// MetricRequestsSuccess used to count the num of successful requests
|
|
|
|
MetricRequestsSuccess = "success"
|
2021-08-13 10:50:09 +08:00
|
|
|
|
|
|
|
// ConnectEtcdMaxRetryTime used to limit the max retry time for connection etcd
|
|
|
|
ConnectEtcdMaxRetryTime = 1000
|
2021-01-24 21:20:11 +08:00
|
|
|
)
|
2021-01-22 09:36:40 +08:00
|
|
|
|
2021-09-09 16:40:00 +08:00
|
|
|
const illegalRequestErrStr = "Illegal request"
|
|
|
|
|
2021-06-29 17:34:13 +08:00
|
|
|
// DataNode communicates with outside services and unioun all
|
|
|
|
// services in datanode package.
|
2021-05-08 15:24:12 +08:00
|
|
|
//
|
2021-06-29 17:34:13 +08:00
|
|
|
// DataNode implements `types.Component`, `types.DataNode` interfaces.
|
2021-09-10 14:46:00 +08:00
|
|
|
// `rootCoord` is a grpc client of root coordinator.
|
|
|
|
// `dataCoord` is a grpc client of data service.
|
2021-06-29 17:34:13 +08:00
|
|
|
// `NodeID` is unique to each datanode.
|
2021-05-25 15:35:37 +08:00
|
|
|
// `State` is current statement of this data node, indicating whether it's healthy.
|
2021-05-08 15:24:12 +08:00
|
|
|
//
|
2021-06-29 17:34:13 +08:00
|
|
|
// `vchan2SyncService` is a map of vchannlName to dataSyncService, so that datanode
|
|
|
|
// has ability to scale flowgraph.
|
|
|
|
// `vchan2FlushCh` holds flush-signal channels for every flowgraph.
|
|
|
|
|
|
|
|
// `clearSignal` is a signal channel for releasing the flowgraph resources.
|
|
|
|
// `segmentCache` stores all flushing and flushed segments.
|
2021-03-05 20:41:34 +08:00
|
|
|
type DataNode struct {
|
2021-06-29 17:34:13 +08:00
|
|
|
ctx context.Context
|
|
|
|
cancel context.CancelFunc
|
|
|
|
NodeID UniqueID
|
|
|
|
Role string
|
|
|
|
State atomic.Value // internalpb.StateCode_Initializing
|
2021-01-24 21:20:11 +08:00
|
|
|
|
2021-05-28 16:47:29 +08:00
|
|
|
chanMut sync.RWMutex
|
2021-06-07 11:25:37 +08:00
|
|
|
vchan2SyncService map[string]*dataSyncService // vchannel name
|
|
|
|
vchan2FlushCh map[string]chan<- *flushMsg // vchannel name
|
|
|
|
clearSignal chan UniqueID // collection ID
|
2021-06-11 17:53:37 +08:00
|
|
|
segmentCache *Cache
|
2021-01-24 21:20:11 +08:00
|
|
|
|
2021-06-21 18:22:13 +08:00
|
|
|
rootCoord types.RootCoord
|
|
|
|
dataCoord types.DataCoord
|
2021-01-24 21:20:11 +08:00
|
|
|
|
2021-08-13 10:50:09 +08:00
|
|
|
session *sessionutil.Session
|
|
|
|
kvClient *etcdkv.EtcdKV
|
2021-05-21 19:28:52 +08:00
|
|
|
|
2021-03-05 20:41:34 +08:00
|
|
|
closer io.Closer
|
2021-02-08 14:30:54 +08:00
|
|
|
|
2021-03-05 20:41:34 +08:00
|
|
|
msFactory msgstream.Factory
|
|
|
|
}
|
2021-01-24 21:20:11 +08:00
|
|
|
|
2021-05-08 15:24:12 +08:00
|
|
|
// NewDataNode will return a DataNode with abnormal state.
|
2021-02-08 14:30:54 +08:00
|
|
|
func NewDataNode(ctx context.Context, factory msgstream.Factory) *DataNode {
|
2021-03-08 15:25:55 +08:00
|
|
|
rand.Seed(time.Now().UnixNano())
|
2021-02-01 11:44:02 +08:00
|
|
|
ctx2, cancel2 := context.WithCancel(ctx)
|
2021-01-19 11:37:16 +08:00
|
|
|
node := &DataNode{
|
2021-06-29 17:34:13 +08:00
|
|
|
ctx: ctx2,
|
|
|
|
cancel: cancel2,
|
|
|
|
Role: typeutil.DataNodeRole,
|
2021-02-04 17:47:19 +08:00
|
|
|
|
2021-06-21 17:28:03 +08:00
|
|
|
rootCoord: nil,
|
2021-06-21 18:22:13 +08:00
|
|
|
dataCoord: nil,
|
2021-06-21 17:28:03 +08:00
|
|
|
msFactory: factory,
|
|
|
|
segmentCache: newCache(),
|
2021-05-25 15:35:37 +08:00
|
|
|
|
|
|
|
vchan2SyncService: make(map[string]*dataSyncService),
|
|
|
|
vchan2FlushCh: make(map[string]chan<- *flushMsg),
|
2021-06-07 11:25:37 +08:00
|
|
|
clearSignal: make(chan UniqueID, 100),
|
2021-01-19 11:37:16 +08:00
|
|
|
}
|
2021-03-12 14:22:09 +08:00
|
|
|
node.UpdateStateCode(internalpb.StateCode_Abnormal)
|
2021-01-19 11:37:16 +08:00
|
|
|
return node
|
|
|
|
}
|
|
|
|
|
2021-06-22 19:08:03 +08:00
|
|
|
// SetRootCoordInterface sets RootCoord's grpc client, error is returned if repeatedly set.
|
2021-06-21 17:28:03 +08:00
|
|
|
func (node *DataNode) SetRootCoordInterface(rc types.RootCoord) error {
|
2021-02-03 17:30:10 +08:00
|
|
|
switch {
|
2021-06-21 17:28:03 +08:00
|
|
|
case rc == nil, node.rootCoord != nil:
|
2021-02-03 17:30:10 +08:00
|
|
|
return errors.New("Nil parameter or repeatly set")
|
|
|
|
default:
|
2021-06-21 17:28:03 +08:00
|
|
|
node.rootCoord = rc
|
2021-02-03 17:30:10 +08:00
|
|
|
return nil
|
|
|
|
}
|
2021-01-26 14:46:54 +08:00
|
|
|
}
|
|
|
|
|
2021-06-21 18:22:13 +08:00
|
|
|
// SetDataCoordInterface sets data service's grpc client, error is returned if repeatedly set.
|
|
|
|
func (node *DataNode) SetDataCoordInterface(ds types.DataCoord) error {
|
2021-02-03 17:30:10 +08:00
|
|
|
switch {
|
2021-06-21 18:22:13 +08:00
|
|
|
case ds == nil, node.dataCoord != nil:
|
2021-02-03 17:30:10 +08:00
|
|
|
return errors.New("Nil parameter or repeatly set")
|
|
|
|
default:
|
2021-06-21 18:22:13 +08:00
|
|
|
node.dataCoord = ds
|
2021-02-03 17:30:10 +08:00
|
|
|
return nil
|
|
|
|
}
|
2021-01-26 14:46:54 +08:00
|
|
|
}
|
|
|
|
|
2021-06-29 17:34:13 +08:00
|
|
|
// Register register datanode to etcd
|
2021-05-25 15:06:05 +08:00
|
|
|
func (node *DataNode) Register() error {
|
2021-06-11 22:04:41 +08:00
|
|
|
node.session = sessionutil.NewSession(node.ctx, Params.MetaRootPath, Params.EtcdEndpoints)
|
2021-05-25 15:06:05 +08:00
|
|
|
node.session.Init(typeutil.DataNodeRole, Params.IP+":"+strconv.Itoa(Params.Port), false)
|
|
|
|
Params.NodeID = node.session.ServerID
|
2021-08-13 10:50:09 +08:00
|
|
|
node.NodeID = node.session.ServerID
|
|
|
|
// Start node watch node
|
|
|
|
go node.StartWatchChannels(node.ctx)
|
2021-06-16 19:03:57 +08:00
|
|
|
|
|
|
|
Params.initMsgChannelSubName()
|
|
|
|
log.Debug("DataNode Init",
|
|
|
|
zap.String("MsgChannelSubName", Params.MsgChannelSubName),
|
|
|
|
)
|
2021-05-25 15:06:05 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-06-29 17:34:13 +08:00
|
|
|
// Init function do nothing now.
|
2021-01-22 09:36:40 +08:00
|
|
|
func (node *DataNode) Init() error {
|
2021-06-08 19:25:37 +08:00
|
|
|
log.Debug("DataNode Init",
|
2021-06-29 17:34:13 +08:00
|
|
|
zap.String("SegmentStatisticsChannelName", Params.SegmentStatisticsChannelName),
|
|
|
|
zap.String("TimeTickChannelName", Params.TimeTickChannelName),
|
2021-06-08 19:25:37 +08:00
|
|
|
)
|
2021-01-24 21:20:11 +08:00
|
|
|
|
2021-05-25 15:35:37 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-08-13 10:50:09 +08:00
|
|
|
// StartWatchChannels start loop to watch channel allocation status via kv(etcd for now)
|
|
|
|
func (node *DataNode) StartWatchChannels(ctx context.Context) {
|
|
|
|
defer logutil.LogPanic()
|
|
|
|
// REF MEP#7 watch path should be [prefix]/channel/{node_id}/{channel_name}
|
|
|
|
watchPrefix := fmt.Sprintf("channel/%d", node.NodeID)
|
|
|
|
evtChan := node.kvClient.WatchWithPrefix(watchPrefix)
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
log.Debug("watch etcd loop quit")
|
|
|
|
return
|
|
|
|
case event := <-evtChan:
|
|
|
|
if event.Canceled { // failed to watch
|
|
|
|
log.Warn("Watch channel failed", zap.Error(event.Err()))
|
|
|
|
// if watch loop return due to event canceled, the datanode is not functional anymore
|
|
|
|
// stop the datanode and wait for restart
|
|
|
|
node.Stop()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
for _, evt := range event.Events {
|
|
|
|
go node.handleChannelEvt(evt)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-09-10 14:46:00 +08:00
|
|
|
// handleChannelEvt handles event from kv watch event
|
2021-08-13 10:50:09 +08:00
|
|
|
func (node *DataNode) handleChannelEvt(evt *clientv3.Event) {
|
|
|
|
switch evt.Type {
|
|
|
|
case clientv3.EventTypePut: // datacoord shall put channels needs to be watched here
|
|
|
|
watchInfo := datapb.ChannelWatchInfo{}
|
|
|
|
err := proto.Unmarshal(evt.Kv.Value, &watchInfo)
|
|
|
|
if err != nil {
|
|
|
|
log.Warn("fail to parse ChannelWatchInfo", zap.String("key", string(evt.Kv.Key)), zap.Error(err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if watchInfo.State == datapb.ChannelWatchState_Complete {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if watchInfo.Vchan == nil {
|
|
|
|
log.Warn("found ChannelWatchInfo with nil VChannelInfo", zap.String("key", string(evt.Kv.Key)))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
err = node.NewDataSyncService(watchInfo.Vchan)
|
|
|
|
if err != nil {
|
|
|
|
log.Warn("fail to create DataSyncService", zap.String("key", string(evt.Kv.Key)), zap.Error(err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
watchInfo.State = datapb.ChannelWatchState_Complete
|
|
|
|
v, _ := proto.Marshal(&watchInfo)
|
|
|
|
err = node.kvClient.Save(fmt.Sprintf("channel/%d/%s", node.NodeID, watchInfo.Vchan.ChannelName), string(v))
|
|
|
|
if err != nil {
|
|
|
|
log.Warn("fail to change WatchState to complete", zap.String("key", string(evt.Kv.Key)), zap.Error(err))
|
|
|
|
node.ReleaseDataSyncService(string(evt.Kv.Key))
|
2021-09-10 14:46:00 +08:00
|
|
|
// TODO GOOSE: maybe retry logic and exit logic
|
2021-08-13 10:50:09 +08:00
|
|
|
}
|
|
|
|
case clientv3.EventTypeDelete:
|
|
|
|
// guaranteed there is no "/" in channel name
|
|
|
|
parts := strings.Split(string(evt.Kv.Key), "/")
|
|
|
|
node.ReleaseDataSyncService(parts[len(parts)-1])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-28 14:54:31 +08:00
|
|
|
// NewDataSyncService adds a new dataSyncService for new dmlVchannel and starts dataSyncService.
|
2021-06-04 09:57:54 +08:00
|
|
|
func (node *DataNode) NewDataSyncService(vchan *datapb.VchannelInfo) error {
|
2021-05-28 16:47:29 +08:00
|
|
|
node.chanMut.Lock()
|
|
|
|
defer node.chanMut.Unlock()
|
2021-06-04 09:57:54 +08:00
|
|
|
if _, ok := node.vchan2SyncService[vchan.GetChannelName()]; ok {
|
2021-05-25 15:35:37 +08:00
|
|
|
return nil
|
2021-05-08 15:24:12 +08:00
|
|
|
}
|
|
|
|
|
2021-06-21 17:28:03 +08:00
|
|
|
replica := newReplica(node.rootCoord, vchan.CollectionID)
|
2021-05-29 18:04:30 +08:00
|
|
|
|
2021-06-21 17:28:03 +08:00
|
|
|
var alloc allocatorInterface = newAllocator(node.rootCoord)
|
2021-02-03 17:30:10 +08:00
|
|
|
|
2021-06-11 09:24:52 +08:00
|
|
|
log.Debug("Received Vchannel Info",
|
|
|
|
zap.Int("Unflushed Segment Number", len(vchan.GetUnflushedSegments())),
|
|
|
|
zap.Int("Flushed Segment Number", len(vchan.GetFlushedSegments())),
|
|
|
|
)
|
|
|
|
|
2021-05-25 15:35:37 +08:00
|
|
|
flushChan := make(chan *flushMsg, 100)
|
2021-06-21 18:22:13 +08:00
|
|
|
dataSyncService, err := newDataSyncService(node.ctx, flushChan, replica, alloc, node.msFactory, vchan, node.clearSignal, node.dataCoord)
|
2021-06-19 15:18:06 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2021-06-29 17:34:13 +08:00
|
|
|
|
2021-06-04 09:57:54 +08:00
|
|
|
node.vchan2SyncService[vchan.GetChannelName()] = dataSyncService
|
|
|
|
node.vchan2FlushCh[vchan.GetChannelName()] = flushChan
|
2021-05-25 15:35:37 +08:00
|
|
|
|
2021-06-29 17:34:13 +08:00
|
|
|
log.Info("Start New dataSyncService",
|
|
|
|
zap.Int64("Collection ID", vchan.GetCollectionID()),
|
|
|
|
zap.String("Vchannel name", vchan.GetChannelName()),
|
|
|
|
)
|
2021-05-25 15:35:37 +08:00
|
|
|
go dataSyncService.start()
|
|
|
|
|
2021-01-24 21:20:11 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-06-07 11:25:37 +08:00
|
|
|
// BackGroundGC runs in background to release datanode resources
|
|
|
|
func (node *DataNode) BackGroundGC(collIDCh <-chan UniqueID) {
|
|
|
|
log.Info("DataNode Background GC Start")
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case collID := <-collIDCh:
|
|
|
|
log.Info("GC collection", zap.Int64("ID", collID))
|
|
|
|
for _, vchanName := range node.getChannelNamesbyCollectionID(collID) {
|
|
|
|
node.ReleaseDataSyncService(vchanName)
|
|
|
|
}
|
|
|
|
case <-node.ctx.Done():
|
2021-06-09 17:31:48 +08:00
|
|
|
log.Info("DataNode ctx done")
|
2021-06-07 11:25:37 +08:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// ReleaseDataSyncService release flowgraph resources for a vchanName
|
|
|
|
func (node *DataNode) ReleaseDataSyncService(vchanName string) {
|
|
|
|
log.Info("Release flowgraph resources begin", zap.String("Vchannel", vchanName))
|
|
|
|
|
|
|
|
node.chanMut.Lock()
|
2021-06-11 17:53:37 +08:00
|
|
|
defer node.chanMut.Unlock()
|
2021-06-07 11:25:37 +08:00
|
|
|
if dss, ok := node.vchan2SyncService[vchanName]; ok {
|
|
|
|
dss.close()
|
|
|
|
}
|
|
|
|
|
|
|
|
delete(node.vchan2SyncService, vchanName)
|
|
|
|
delete(node.vchan2FlushCh, vchanName)
|
|
|
|
|
|
|
|
log.Debug("Release flowgraph resources end", zap.String("Vchannel", vchanName))
|
|
|
|
}
|
|
|
|
|
2021-06-07 13:58:37 +08:00
|
|
|
var FilterThreshold Timestamp
|
|
|
|
|
2021-05-28 14:54:31 +08:00
|
|
|
// Start will update DataNode state to HEALTHY
|
2021-01-24 21:20:11 +08:00
|
|
|
func (node *DataNode) Start() error {
|
2021-06-07 13:58:37 +08:00
|
|
|
|
2021-06-22 16:14:09 +08:00
|
|
|
rep, err := node.rootCoord.AllocTimestamp(node.ctx, &rootcoordpb.AllocTimestampRequest{
|
2021-06-07 13:58:37 +08:00
|
|
|
Base: &commonpb.MsgBase{
|
|
|
|
MsgType: commonpb.MsgType_RequestTSO,
|
|
|
|
MsgID: 0,
|
|
|
|
Timestamp: 0,
|
|
|
|
SourceID: node.NodeID,
|
|
|
|
},
|
|
|
|
Count: 1,
|
|
|
|
})
|
2021-08-13 10:50:09 +08:00
|
|
|
if err != nil {
|
|
|
|
log.Warn("fail to alloc timestamp", zap.Error(err))
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
connectEtcdFn := func() error {
|
2021-08-13 14:00:14 +08:00
|
|
|
etcdKV, err := etcdkv.NewEtcdKV(Params.EtcdEndpoints, Params.MetaRootPath)
|
2021-08-13 10:50:09 +08:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2021-08-13 14:00:14 +08:00
|
|
|
node.kvClient = etcdKV
|
2021-08-13 10:50:09 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
err = retry.Do(node.ctx, connectEtcdFn, retry.Attempts(ConnectEtcdMaxRetryTime))
|
|
|
|
if err != nil {
|
|
|
|
return errors.New("DataNode fail to connect etcd")
|
|
|
|
}
|
2021-06-07 13:58:37 +08:00
|
|
|
|
|
|
|
if rep.Status.ErrorCode != commonpb.ErrorCode_Success || err != nil {
|
|
|
|
return errors.New("DataNode fail to start")
|
|
|
|
}
|
|
|
|
|
|
|
|
FilterThreshold = rep.GetTimestamp()
|
|
|
|
|
2021-06-07 11:25:37 +08:00
|
|
|
go node.BackGroundGC(node.clearSignal)
|
2021-03-12 14:22:09 +08:00
|
|
|
node.UpdateStateCode(internalpb.StateCode_Healthy)
|
2021-02-04 17:47:19 +08:00
|
|
|
return nil
|
2021-01-19 11:37:16 +08:00
|
|
|
}
|
|
|
|
|
2021-05-28 14:54:31 +08:00
|
|
|
// UpdateStateCode updates datanode's state code
|
2021-03-12 14:22:09 +08:00
|
|
|
func (node *DataNode) UpdateStateCode(code internalpb.StateCode) {
|
2021-02-23 11:40:30 +08:00
|
|
|
node.State.Store(code)
|
|
|
|
}
|
|
|
|
|
2021-09-01 10:13:15 +08:00
|
|
|
func (node *DataNode) isHealthy() bool {
|
|
|
|
code := node.State.Load().(internalpb.StateCode)
|
|
|
|
return code == internalpb.StateCode_Healthy
|
|
|
|
}
|
|
|
|
|
2021-05-28 14:54:31 +08:00
|
|
|
// WatchDmChannels create a new dataSyncService for every unique dmlVchannel name, ignore if dmlVchannel existed.
|
2021-03-12 14:22:09 +08:00
|
|
|
func (node *DataNode) WatchDmChannels(ctx context.Context, in *datapb.WatchDmChannelsRequest) (*commonpb.Status, error) {
|
2021-06-21 18:08:15 +08:00
|
|
|
metrics.DataNodeWatchDmChannelsCounter.WithLabelValues(MetricRequestsTotal).Inc()
|
2021-02-03 17:30:10 +08:00
|
|
|
status := &commonpb.Status{
|
2021-03-10 22:06:22 +08:00
|
|
|
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
2021-02-03 17:30:10 +08:00
|
|
|
}
|
2021-01-26 14:46:54 +08:00
|
|
|
|
2021-02-03 17:30:10 +08:00
|
|
|
switch {
|
2021-09-09 16:40:00 +08:00
|
|
|
case !node.isHealthy():
|
|
|
|
status.Reason = msgDataNodeIsUnhealthy(node.NodeID)
|
2021-06-21 18:08:15 +08:00
|
|
|
return status, nil
|
2021-02-03 17:30:10 +08:00
|
|
|
|
2021-05-25 15:35:37 +08:00
|
|
|
case len(in.GetVchannels()) == 0:
|
2021-09-09 16:40:00 +08:00
|
|
|
status.Reason = illegalRequestErrStr
|
2021-06-21 18:08:15 +08:00
|
|
|
return status, nil
|
2021-02-03 17:30:10 +08:00
|
|
|
|
|
|
|
default:
|
2021-06-07 13:58:37 +08:00
|
|
|
for _, chanInfo := range in.GetVchannels() {
|
2021-06-08 19:25:37 +08:00
|
|
|
log.Info("DataNode new dataSyncService",
|
|
|
|
zap.String("channel name", chanInfo.ChannelName),
|
|
|
|
zap.Any("channal Info", chanInfo),
|
|
|
|
)
|
2021-06-19 15:18:06 +08:00
|
|
|
if err := node.NewDataSyncService(chanInfo); err != nil {
|
|
|
|
log.Warn("Failed to new data sync service", zap.Any("channel", chanInfo))
|
|
|
|
}
|
2021-05-25 15:35:37 +08:00
|
|
|
}
|
|
|
|
|
2021-03-10 22:06:22 +08:00
|
|
|
status.ErrorCode = commonpb.ErrorCode_Success
|
2021-06-08 19:25:37 +08:00
|
|
|
log.Debug("DataNode WatchDmChannels Done")
|
2021-06-21 18:08:15 +08:00
|
|
|
metrics.DataNodeWatchDmChannelsCounter.WithLabelValues(MetricRequestsSuccess).Inc()
|
2021-02-03 17:30:10 +08:00
|
|
|
return status, nil
|
|
|
|
}
|
2021-01-22 09:36:40 +08:00
|
|
|
}
|
|
|
|
|
2021-05-29 18:04:30 +08:00
|
|
|
// GetComponentStates will return current state of DataNode
|
2021-03-12 14:22:09 +08:00
|
|
|
func (node *DataNode) GetComponentStates(ctx context.Context) (*internalpb.ComponentStates, error) {
|
2021-02-26 10:13:36 +08:00
|
|
|
log.Debug("DataNode current state", zap.Any("State", node.State.Load()))
|
2021-03-12 14:22:09 +08:00
|
|
|
states := &internalpb.ComponentStates{
|
|
|
|
State: &internalpb.ComponentInfo{
|
2021-01-24 21:20:11 +08:00
|
|
|
NodeID: Params.NodeID,
|
|
|
|
Role: node.Role,
|
2021-03-12 14:22:09 +08:00
|
|
|
StateCode: node.State.Load().(internalpb.StateCode),
|
2021-01-24 21:20:11 +08:00
|
|
|
},
|
2021-03-12 14:22:09 +08:00
|
|
|
SubcomponentStates: make([]*internalpb.ComponentInfo, 0),
|
2021-03-10 22:06:22 +08:00
|
|
|
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
|
2021-01-24 21:20:11 +08:00
|
|
|
}
|
|
|
|
return states, nil
|
2021-01-22 09:36:40 +08:00
|
|
|
}
|
|
|
|
|
2021-06-07 11:25:37 +08:00
|
|
|
func (node *DataNode) getChannelNamebySegmentID(segID UniqueID) string {
|
2021-05-28 16:47:29 +08:00
|
|
|
node.chanMut.RLock()
|
|
|
|
defer node.chanMut.RUnlock()
|
2021-05-25 15:35:37 +08:00
|
|
|
for name, dataSync := range node.vchan2SyncService {
|
2021-08-11 14:24:09 +08:00
|
|
|
if dataSync.replica.hasSegment(segID, false) {
|
2021-05-25 15:35:37 +08:00
|
|
|
return name
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
|
2021-06-07 11:25:37 +08:00
|
|
|
func (node *DataNode) getChannelNamesbyCollectionID(collID UniqueID) []string {
|
|
|
|
node.chanMut.RLock()
|
|
|
|
defer node.chanMut.RUnlock()
|
|
|
|
|
|
|
|
channels := make([]string, 0, len(node.vchan2SyncService))
|
|
|
|
for name, dataSync := range node.vchan2SyncService {
|
|
|
|
if dataSync.collectionID == collID {
|
|
|
|
channels = append(channels, name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return channels
|
|
|
|
}
|
|
|
|
|
2021-05-29 18:04:30 +08:00
|
|
|
// ReadyToFlush tells wether DataNode is ready for flushing
|
|
|
|
func (node *DataNode) ReadyToFlush() error {
|
|
|
|
if node.State.Load().(internalpb.StateCode) != internalpb.StateCode_Healthy {
|
|
|
|
return errors.New("DataNode not in HEALTHY state")
|
|
|
|
}
|
|
|
|
|
|
|
|
node.chanMut.RLock()
|
|
|
|
defer node.chanMut.RUnlock()
|
|
|
|
if len(node.vchan2SyncService) == 0 && len(node.vchan2FlushCh) == 0 {
|
|
|
|
// Healthy but Idle
|
|
|
|
msg := "DataNode HEALTHY but IDLE, please try WatchDmChannels to make it work"
|
2021-06-11 17:53:37 +08:00
|
|
|
log.Warn(msg)
|
2021-05-29 18:04:30 +08:00
|
|
|
return errors.New(msg)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(node.vchan2SyncService) != len(node.vchan2FlushCh) {
|
|
|
|
// TODO restart
|
|
|
|
msg := "DataNode HEALTHY but abnormal inside, restarting..."
|
2021-06-11 17:53:37 +08:00
|
|
|
log.Warn(msg)
|
2021-05-29 18:04:30 +08:00
|
|
|
return errors.New(msg)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-05-08 15:24:12 +08:00
|
|
|
// FlushSegments packs flush messages into flowgraph through flushChan.
|
2021-05-28 14:54:31 +08:00
|
|
|
// If DataNode receives a valid segment to flush, new flush message for the segment should be ignored.
|
|
|
|
// So if receiving calls to flush segment A, DataNode should guarantee the segment to be flushed.
|
2021-05-29 18:04:30 +08:00
|
|
|
//
|
|
|
|
// There are 1 precondition: The segmentID in req is in ascending order.
|
2021-03-12 14:22:09 +08:00
|
|
|
func (node *DataNode) FlushSegments(ctx context.Context, req *datapb.FlushSegmentsRequest) (*commonpb.Status, error) {
|
2021-06-21 18:08:15 +08:00
|
|
|
metrics.DataNodeFlushSegmentsCounter.WithLabelValues(MetricRequestsTotal).Inc()
|
2021-05-25 15:35:37 +08:00
|
|
|
status := &commonpb.Status{
|
|
|
|
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
2021-01-26 14:46:54 +08:00
|
|
|
}
|
|
|
|
|
2021-05-29 18:04:30 +08:00
|
|
|
if err := node.ReadyToFlush(); err != nil {
|
|
|
|
status.Reason = err.Error()
|
|
|
|
return status, nil
|
|
|
|
}
|
|
|
|
|
2021-06-11 17:53:37 +08:00
|
|
|
numOfFlushingSeg := len(req.SegmentIDs)
|
2021-06-29 17:34:13 +08:00
|
|
|
log.Debug("FlushSegments ...",
|
|
|
|
zap.Int("num", len(req.SegmentIDs)),
|
|
|
|
zap.Int64s("segments", req.SegmentIDs),
|
|
|
|
)
|
|
|
|
|
2021-08-19 13:00:12 +08:00
|
|
|
dmlFlushedCh := make(chan []*datapb.FieldBinlog, len(req.SegmentIDs))
|
2021-05-25 15:35:37 +08:00
|
|
|
for _, id := range req.SegmentIDs {
|
2021-06-07 11:25:37 +08:00
|
|
|
chanName := node.getChannelNamebySegmentID(id)
|
2021-08-11 14:24:09 +08:00
|
|
|
log.Debug("vchannel",
|
|
|
|
zap.String("name", chanName),
|
|
|
|
zap.Int64("SegmentID", id))
|
|
|
|
|
2021-05-29 18:04:30 +08:00
|
|
|
if len(chanName) == 0 {
|
2021-05-25 15:35:37 +08:00
|
|
|
status.Reason = fmt.Sprintf("DataNode not find segment %d!", id)
|
2021-08-03 09:59:24 +08:00
|
|
|
return status, nil
|
2021-05-25 15:35:37 +08:00
|
|
|
}
|
2021-05-29 18:04:30 +08:00
|
|
|
|
2021-06-11 17:53:37 +08:00
|
|
|
if node.segmentCache.checkIfCached(id) {
|
|
|
|
// Segment in flushing or flushed, ignore
|
|
|
|
log.Info("Segment in flushing, ignore it", zap.Int64("ID", id))
|
|
|
|
numOfFlushingSeg--
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
node.segmentCache.Cache(id)
|
|
|
|
|
2021-05-28 16:47:29 +08:00
|
|
|
node.chanMut.RLock()
|
2021-05-25 15:35:37 +08:00
|
|
|
flushCh, ok := node.vchan2FlushCh[chanName]
|
2021-05-28 16:47:29 +08:00
|
|
|
node.chanMut.RUnlock()
|
2021-05-25 15:35:37 +08:00
|
|
|
if !ok {
|
2021-05-28 14:54:31 +08:00
|
|
|
// TODO restart DataNode or reshape vchan2FlushCh and vchan2SyncService
|
2021-05-29 18:04:30 +08:00
|
|
|
status.Reason = "DataNode abnormal, restarting"
|
|
|
|
return status, nil
|
2021-05-25 15:35:37 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
flushmsg := &flushMsg{
|
|
|
|
msgID: req.Base.MsgID,
|
|
|
|
timestamp: req.Base.Timestamp,
|
|
|
|
segmentID: id,
|
|
|
|
collectionID: req.CollectionID,
|
|
|
|
dmlFlushedCh: dmlFlushedCh,
|
|
|
|
}
|
2021-06-04 16:31:34 +08:00
|
|
|
flushCh <- flushmsg
|
2021-05-25 15:35:37 +08:00
|
|
|
|
2021-06-04 16:31:34 +08:00
|
|
|
}
|
2021-06-11 17:53:37 +08:00
|
|
|
|
2021-06-04 16:31:34 +08:00
|
|
|
failedSegments := ""
|
2021-06-11 17:53:37 +08:00
|
|
|
for i := 0; i < numOfFlushingSeg; i++ {
|
2021-06-04 16:31:34 +08:00
|
|
|
msg := <-dmlFlushedCh
|
|
|
|
if len(msg) != 1 {
|
|
|
|
panic("flush size expect to 1")
|
2021-05-25 15:35:37 +08:00
|
|
|
}
|
2021-08-19 13:00:12 +08:00
|
|
|
if msg[0].Binlogs == nil {
|
|
|
|
failedSegments += fmt.Sprintf(" %d", msg[0].FieldID)
|
2021-05-29 18:04:30 +08:00
|
|
|
}
|
2021-06-04 16:31:34 +08:00
|
|
|
}
|
|
|
|
if len(failedSegments) != 0 {
|
|
|
|
status.Reason = fmt.Sprintf("flush failed segment list = %s", failedSegments)
|
|
|
|
return status, nil
|
2021-05-25 15:35:37 +08:00
|
|
|
}
|
2021-08-11 14:24:09 +08:00
|
|
|
|
|
|
|
node.segmentCache.Remove(req.SegmentIDs...)
|
|
|
|
log.Debug("FlushSegments Done",
|
|
|
|
zap.Int64s("segments", req.SegmentIDs))
|
2021-05-25 15:35:37 +08:00
|
|
|
|
|
|
|
status.ErrorCode = commonpb.ErrorCode_Success
|
2021-06-21 18:08:15 +08:00
|
|
|
metrics.DataNodeFlushSegmentsCounter.WithLabelValues(MetricRequestsSuccess).Inc()
|
2021-05-25 15:35:37 +08:00
|
|
|
return status, nil
|
2021-01-22 09:36:40 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (node *DataNode) Stop() error {
|
2021-02-01 11:44:02 +08:00
|
|
|
node.cancel()
|
2021-01-19 11:37:16 +08:00
|
|
|
|
2021-05-28 16:47:29 +08:00
|
|
|
node.chanMut.RLock()
|
|
|
|
defer node.chanMut.RUnlock()
|
2021-01-19 11:37:16 +08:00
|
|
|
// close services
|
2021-05-25 15:35:37 +08:00
|
|
|
for _, syncService := range node.vchan2SyncService {
|
|
|
|
if syncService != nil {
|
|
|
|
(*syncService).close()
|
|
|
|
}
|
2021-01-19 11:37:16 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if node.closer != nil {
|
|
|
|
node.closer.Close()
|
|
|
|
}
|
2021-01-22 09:36:40 +08:00
|
|
|
return nil
|
2021-01-26 14:46:54 +08:00
|
|
|
}
|
|
|
|
|
2021-02-26 17:44:24 +08:00
|
|
|
func (node *DataNode) GetTimeTickChannel(ctx context.Context) (*milvuspb.StringResponse, error) {
|
|
|
|
return &milvuspb.StringResponse{
|
|
|
|
Status: &commonpb.Status{
|
2021-03-10 22:06:22 +08:00
|
|
|
ErrorCode: commonpb.ErrorCode_Success,
|
2021-02-26 17:44:24 +08:00
|
|
|
Reason: "",
|
|
|
|
},
|
|
|
|
Value: "",
|
|
|
|
}, nil
|
2021-01-26 14:46:54 +08:00
|
|
|
}
|
2021-01-19 11:37:16 +08:00
|
|
|
|
2021-02-26 17:44:24 +08:00
|
|
|
func (node *DataNode) GetStatisticsChannel(ctx context.Context) (*milvuspb.StringResponse, error) {
|
|
|
|
return &milvuspb.StringResponse{
|
|
|
|
Status: &commonpb.Status{
|
2021-03-10 22:06:22 +08:00
|
|
|
ErrorCode: commonpb.ErrorCode_Success,
|
2021-02-26 17:44:24 +08:00
|
|
|
Reason: "",
|
|
|
|
},
|
|
|
|
Value: "",
|
|
|
|
}, nil
|
2021-01-19 11:37:16 +08:00
|
|
|
}
|
2021-09-01 10:13:15 +08:00
|
|
|
|
|
|
|
// TODO(dragondriver): cache the Metrics and set a retention to the cache
|
|
|
|
func (node *DataNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) {
|
|
|
|
log.Debug("DataNode.GetMetrics",
|
|
|
|
zap.Int64("node_id", Params.NodeID),
|
|
|
|
zap.String("req", req.Request))
|
|
|
|
|
|
|
|
if !node.isHealthy() {
|
|
|
|
log.Warn("DataNode.GetMetrics failed",
|
|
|
|
zap.Int64("node_id", Params.NodeID),
|
|
|
|
zap.String("req", req.Request),
|
|
|
|
zap.Error(errDataNodeIsUnhealthy(Params.NodeID)))
|
|
|
|
|
|
|
|
return &milvuspb.GetMetricsResponse{
|
|
|
|
Status: &commonpb.Status{
|
|
|
|
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
|
|
|
Reason: msgDataNodeIsUnhealthy(Params.NodeID),
|
|
|
|
},
|
|
|
|
Response: "",
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
metricType, err := metricsinfo.ParseMetricType(req.Request)
|
|
|
|
if err != nil {
|
|
|
|
log.Warn("DataNode.GetMetrics failed to parse metric type",
|
|
|
|
zap.Int64("node_id", Params.NodeID),
|
|
|
|
zap.String("req", req.Request),
|
|
|
|
zap.Error(err))
|
|
|
|
|
|
|
|
return &milvuspb.GetMetricsResponse{
|
|
|
|
Status: &commonpb.Status{
|
|
|
|
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
|
|
|
Reason: err.Error(),
|
|
|
|
},
|
|
|
|
Response: "",
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Debug("DataNode.GetMetrics",
|
|
|
|
zap.String("metric_type", metricType))
|
|
|
|
|
|
|
|
if metricType == metricsinfo.SystemInfoMetrics {
|
|
|
|
systemInfoMetrics, err := node.getSystemInfoMetrics(ctx, req)
|
|
|
|
|
|
|
|
log.Debug("DataNode.GetMetrics",
|
|
|
|
zap.Int64("node_id", Params.NodeID),
|
|
|
|
zap.String("req", req.Request),
|
|
|
|
zap.String("metric_type", metricType),
|
|
|
|
zap.Any("systemInfoMetrics", systemInfoMetrics), // TODO(dragondriver): necessary? may be very large
|
|
|
|
zap.Error(err))
|
|
|
|
|
|
|
|
return systemInfoMetrics, err
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Debug("DataNode.GetMetrics failed, request metric type is not implemented yet",
|
|
|
|
zap.Int64("node_id", Params.NodeID),
|
|
|
|
zap.String("req", req.Request),
|
|
|
|
zap.String("metric_type", metricType))
|
|
|
|
|
|
|
|
return &milvuspb.GetMetricsResponse{
|
|
|
|
Status: &commonpb.Status{
|
|
|
|
ErrorCode: commonpb.ErrorCode_UnexpectedError,
|
|
|
|
Reason: metricsinfo.MsgUnimplementedMetric,
|
|
|
|
},
|
|
|
|
Response: "",
|
|
|
|
}, nil
|
|
|
|
}
|