2022-11-10 22:13:04 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package datanode
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"reflect"
|
|
|
|
"time"
|
|
|
|
|
2023-09-18 19:35:29 +08:00
|
|
|
"go.uber.org/atomic"
|
2022-11-10 22:13:04 +08:00
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2023-06-09 01:28:37 +08:00
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
2023-11-15 15:24:18 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/datanode/metacache"
|
|
|
|
"github.com/milvus-io/milvus/internal/datanode/writebuffer"
|
2022-11-10 22:13:04 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/flowgraph"
|
2023-04-06 19:14:32 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/tsoutil"
|
2022-11-10 22:13:04 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
// make sure ttNode implements flowgraph.Node
|
|
|
|
var _ flowgraph.Node = (*ttNode)(nil)
|
|
|
|
|
|
|
|
type ttNode struct {
|
|
|
|
BaseNode
|
2023-11-15 15:24:18 +08:00
|
|
|
vChannelName string
|
|
|
|
metacache metacache.MetaCache
|
2023-11-16 00:22:20 +08:00
|
|
|
writeBufferManager writebuffer.BufferManager
|
2023-11-15 15:24:18 +08:00
|
|
|
lastUpdateTime *atomic.Time
|
2023-12-12 13:48:42 +08:00
|
|
|
cpUpdater *channelCheckpointUpdater
|
2023-12-15 16:04:45 +08:00
|
|
|
dropMode *atomic.Bool
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Name returns node name, implementing flowgraph.Node
|
|
|
|
func (ttn *ttNode) Name() string {
|
|
|
|
return fmt.Sprintf("ttNode-%s", ttn.vChannelName)
|
|
|
|
}
|
|
|
|
|
2023-01-06 14:49:36 +08:00
|
|
|
func (ttn *ttNode) IsValidInMsg(in []Msg) bool {
|
|
|
|
if !ttn.BaseNode.IsValidInMsg(in) {
|
|
|
|
return false
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
2023-01-06 14:49:36 +08:00
|
|
|
_, ok := in[0].(*flowGraphMsg)
|
2022-11-10 22:13:04 +08:00
|
|
|
if !ok {
|
|
|
|
log.Warn("type assertion failed for flowGraphMsg", zap.String("name", reflect.TypeOf(in[0]).Name()))
|
2023-01-06 14:49:36 +08:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2023-10-24 18:22:10 +08:00
|
|
|
func (ttn *ttNode) Close() {
|
|
|
|
}
|
|
|
|
|
2023-01-06 14:49:36 +08:00
|
|
|
// Operate handles input messages, implementing flowgraph.Node
|
|
|
|
func (ttn *ttNode) Operate(in []Msg) []Msg {
|
|
|
|
fgMsg := in[0].(*flowGraphMsg)
|
2023-12-15 16:04:45 +08:00
|
|
|
if fgMsg.dropCollection {
|
|
|
|
ttn.dropMode.Store(true)
|
|
|
|
}
|
|
|
|
|
|
|
|
// skip updating checkpoint for drop collection
|
|
|
|
// even if its the close msg
|
|
|
|
if ttn.dropMode.Load() {
|
|
|
|
log.RatedInfo(1.0, "ttnode in dropMode", zap.String("channel", ttn.vChannelName))
|
|
|
|
return []Msg{}
|
|
|
|
}
|
|
|
|
|
2023-09-18 19:35:29 +08:00
|
|
|
curTs, _ := tsoutil.ParseTS(fgMsg.timeRange.timestampMax)
|
2023-01-06 14:49:36 +08:00
|
|
|
if fgMsg.IsCloseMsg() {
|
2023-03-09 14:13:52 +08:00
|
|
|
if len(fgMsg.endPositions) > 0 {
|
2023-11-15 15:24:18 +08:00
|
|
|
channelPos, _, err := ttn.writeBufferManager.GetCheckpoint(ttn.vChannelName)
|
|
|
|
if err != nil {
|
|
|
|
log.Warn("channel removed", zap.String("channel", ttn.vChannelName), zap.Error(err))
|
|
|
|
return []Msg{}
|
|
|
|
}
|
2023-03-09 14:13:52 +08:00
|
|
|
log.Info("flowgraph is closing, force update channel CP",
|
2023-09-12 21:07:19 +08:00
|
|
|
zap.Time("cpTs", tsoutil.PhysicalTime(channelPos.GetTimestamp())),
|
|
|
|
zap.String("channel", channelPos.GetChannelName()))
|
2023-09-18 19:35:29 +08:00
|
|
|
ttn.updateChannelCP(channelPos, curTs)
|
2023-03-09 14:13:52 +08:00
|
|
|
}
|
2023-01-06 14:49:36 +08:00
|
|
|
return in
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
|
|
|
|
2023-09-18 19:35:29 +08:00
|
|
|
// Do not block and async updateCheckPoint
|
2023-11-15 15:24:18 +08:00
|
|
|
channelPos, needUpdate, err := ttn.writeBufferManager.GetCheckpoint(ttn.vChannelName)
|
|
|
|
if err != nil {
|
|
|
|
log.Warn("channel removed", zap.String("channel", ttn.vChannelName), zap.Error(err))
|
|
|
|
return []Msg{}
|
|
|
|
}
|
2023-09-18 19:35:29 +08:00
|
|
|
nonBlockingNotify := func() {
|
2023-12-12 13:48:42 +08:00
|
|
|
ttn.updateChannelCP(channelPos, curTs)
|
2023-09-12 21:07:19 +08:00
|
|
|
}
|
|
|
|
|
2023-11-15 15:24:18 +08:00
|
|
|
if needUpdate || curTs.Sub(ttn.lastUpdateTime.Load()) >= updateChanCPInterval {
|
2023-09-18 19:35:29 +08:00
|
|
|
nonBlockingNotify()
|
|
|
|
return []Msg{}
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return []Msg{}
|
|
|
|
}
|
|
|
|
|
2023-09-18 19:35:29 +08:00
|
|
|
func (ttn *ttNode) updateChannelCP(channelPos *msgpb.MsgPosition, curTs time.Time) error {
|
2023-12-12 13:48:42 +08:00
|
|
|
callBack := func() error {
|
|
|
|
channelCPTs, _ := tsoutil.ParseTS(channelPos.GetTimestamp())
|
|
|
|
ttn.lastUpdateTime.Store(curTs)
|
|
|
|
ttn.writeBufferManager.NotifyCheckpointUpdated(ttn.vChannelName, channelPos.GetTimestamp())
|
|
|
|
log.Debug("UpdateChannelCheckpoint success",
|
|
|
|
zap.String("channel", ttn.vChannelName),
|
|
|
|
zap.Uint64("cpTs", channelPos.GetTimestamp()),
|
|
|
|
zap.Time("cpTime", channelCPTs))
|
|
|
|
return nil
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
|
|
|
|
2023-12-12 13:48:42 +08:00
|
|
|
err := ttn.cpUpdater.updateChannelCP(channelPos, callBack)
|
|
|
|
return err
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
|
|
|
|
2023-12-12 13:48:42 +08:00
|
|
|
func newTTNode(config *nodeConfig, wbManager writebuffer.BufferManager, cpUpdater *channelCheckpointUpdater) (*ttNode, error) {
|
2022-11-10 22:13:04 +08:00
|
|
|
baseNode := BaseNode{}
|
2022-12-07 18:01:19 +08:00
|
|
|
baseNode.SetMaxQueueLength(Params.DataNodeCfg.FlowGraphMaxQueueLength.GetAsInt32())
|
|
|
|
baseNode.SetMaxParallelism(Params.DataNodeCfg.FlowGraphMaxParallelism.GetAsInt32())
|
2022-11-10 22:13:04 +08:00
|
|
|
|
|
|
|
tt := &ttNode{
|
2023-11-15 15:24:18 +08:00
|
|
|
BaseNode: baseNode,
|
|
|
|
vChannelName: config.vChannelName,
|
|
|
|
metacache: config.metacache,
|
|
|
|
writeBufferManager: wbManager,
|
|
|
|
lastUpdateTime: atomic.NewTime(time.Time{}), // set to Zero to update channel checkpoint immediately after fg started
|
2023-12-12 13:48:42 +08:00
|
|
|
cpUpdater: cpUpdater,
|
2023-12-15 16:04:45 +08:00
|
|
|
dropMode: atomic.NewBool(false),
|
2022-11-10 22:13:04 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return tt, nil
|
|
|
|
}
|