2021-10-15 18:07:09 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
2021-04-19 15:16:33 +08:00
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
2021-10-15 18:07:09 +08:00
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
2021-04-19 15:16:33 +08:00
|
|
|
//
|
2021-10-15 18:07:09 +08:00
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2021-04-19 15:16:33 +08:00
|
|
|
|
2021-01-19 11:37:16 +08:00
|
|
|
package datanode
|
|
|
|
|
|
|
|
import (
|
2023-09-08 09:51:17 +08:00
|
|
|
"context"
|
2021-10-11 21:54:33 +08:00
|
|
|
"fmt"
|
2021-10-15 20:31:16 +08:00
|
|
|
"time"
|
2021-01-19 11:37:16 +08:00
|
|
|
|
2022-11-03 15:03:35 +08:00
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2023-06-09 01:28:37 +08:00
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/flowgraph"
|
2023-04-06 19:14:32 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
|
|
"github.com/milvus-io/milvus/pkg/metrics"
|
|
|
|
"github.com/milvus-io/milvus/pkg/mq/msgdispatcher"
|
|
|
|
"github.com/milvus-io/milvus/pkg/mq/msgstream"
|
|
|
|
"github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/tsoutil"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
2021-01-19 11:37:16 +08:00
|
|
|
)
|
|
|
|
|
2021-10-06 23:02:08 +08:00
|
|
|
// DmInputNode receives messages from message streams, packs messages between two timeticks, and passes all
|
2023-02-26 11:31:49 +08:00
|
|
|
//
|
2023-04-03 16:44:23 +08:00
|
|
|
// messages between two timeticks to the following flowgraph node. In DataNode, the following flow graph node is
|
|
|
|
// flowgraph ddNode.
|
2023-09-20 16:03:23 +08:00
|
|
|
func newDmInputNode(initCtx context.Context, dispatcherClient msgdispatcher.Client, seekPos *msgpb.MsgPosition, dmNodeConfig *nodeConfig) (*flowgraph.InputNode, error) {
|
2023-02-13 16:38:33 +08:00
|
|
|
log := log.With(zap.Int64("nodeID", paramtable.GetNodeID()),
|
2023-07-14 15:56:31 +08:00
|
|
|
zap.Int64("collectionID", dmNodeConfig.collectionID),
|
2023-02-13 16:38:33 +08:00
|
|
|
zap.String("vchannel", dmNodeConfig.vChannelName))
|
|
|
|
var err error
|
|
|
|
var input <-chan *msgstream.MsgPack
|
|
|
|
if seekPos != nil && len(seekPos.MsgID) != 0 {
|
2023-09-20 16:03:23 +08:00
|
|
|
input, err = dispatcherClient.Register(initCtx, dmNodeConfig.vChannelName, seekPos, mqwrapper.SubscriptionPositionUnknown)
|
2021-10-08 19:25:00 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2023-02-13 16:38:33 +08:00
|
|
|
log.Info("datanode seek successfully when register to msgDispatcher",
|
|
|
|
zap.ByteString("msgID", seekPos.GetMsgID()),
|
|
|
|
zap.Time("tsTime", tsoutil.PhysicalTime(seekPos.GetTimestamp())),
|
|
|
|
zap.Duration("tsLag", time.Since(tsoutil.PhysicalTime(seekPos.GetTimestamp()))))
|
2022-10-25 13:23:30 +08:00
|
|
|
} else {
|
2023-09-20 16:03:23 +08:00
|
|
|
input, err = dispatcherClient.Register(initCtx, dmNodeConfig.vChannelName, nil, mqwrapper.SubscriptionPositionEarliest)
|
2023-02-13 16:38:33 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
log.Info("datanode consume successfully when register to msgDispatcher")
|
2021-06-08 19:25:37 +08:00
|
|
|
}
|
2021-01-19 11:37:16 +08:00
|
|
|
|
2022-04-27 10:01:47 +08:00
|
|
|
name := fmt.Sprintf("dmInputNode-data-%d-%s", dmNodeConfig.collectionID, dmNodeConfig.vChannelName)
|
2023-08-21 11:16:20 +08:00
|
|
|
node := flowgraph.NewInputNode(
|
|
|
|
input,
|
|
|
|
name,
|
2023-09-27 11:07:25 +08:00
|
|
|
Params.DataNodeCfg.FlowGraphMaxQueueLength.GetAsInt32(),
|
|
|
|
Params.DataNodeCfg.FlowGraphMaxParallelism.GetAsInt32(),
|
2023-08-21 11:16:20 +08:00
|
|
|
typeutil.DataNodeRole,
|
2023-12-22 14:20:43 +08:00
|
|
|
dmNodeConfig.serverID,
|
2023-08-21 11:16:20 +08:00
|
|
|
dmNodeConfig.collectionID,
|
|
|
|
metrics.AllLabel,
|
|
|
|
)
|
2021-09-23 18:31:55 +08:00
|
|
|
return node, nil
|
2021-01-19 11:37:16 +08:00
|
|
|
}
|