2021-11-04 19:13:50 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
2021-04-19 11:30:19 +08:00
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
2021-11-04 19:13:50 +08:00
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
2021-04-19 11:30:19 +08:00
|
|
|
//
|
2021-11-04 19:13:50 +08:00
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2021-04-19 11:30:19 +08:00
|
|
|
|
2020-10-29 20:42:47 +08:00
|
|
|
package msgstream
|
|
|
|
|
|
|
|
import (
|
2021-02-04 14:37:12 +08:00
|
|
|
"context"
|
|
|
|
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/internalpb"
|
2021-09-27 14:10:09 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/mqclient"
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/typeutil"
|
2020-11-03 14:53:36 +08:00
|
|
|
)
|
2020-10-29 20:42:47 +08:00
|
|
|
|
2021-09-27 23:52:02 +08:00
|
|
|
// UniqueID is an alias for short
|
2020-11-04 17:58:43 +08:00
|
|
|
type UniqueID = typeutil.UniqueID
|
2021-09-27 23:52:02 +08:00
|
|
|
|
|
|
|
// Timestamp is an alias for short
|
2020-11-04 17:58:43 +08:00
|
|
|
type Timestamp = typeutil.Timestamp
|
2021-09-27 23:52:02 +08:00
|
|
|
|
|
|
|
// IntPrimaryKey is an alias for short
|
2020-11-04 17:58:43 +08:00
|
|
|
type IntPrimaryKey = typeutil.IntPrimaryKey
|
2021-09-27 23:52:02 +08:00
|
|
|
|
|
|
|
// MsgPosition is an alias for short
|
2021-03-12 14:22:09 +08:00
|
|
|
type MsgPosition = internalpb.MsgPosition
|
2021-09-27 23:52:02 +08:00
|
|
|
|
|
|
|
// MessageID is an alias for short
|
2021-09-27 14:10:09 +08:00
|
|
|
type MessageID = mqclient.MessageID
|
2020-11-04 17:58:43 +08:00
|
|
|
|
2021-09-23 21:57:55 +08:00
|
|
|
// MsgPack represents a batch of msg in msgstream
|
2020-10-29 20:42:47 +08:00
|
|
|
type MsgPack struct {
|
2021-01-20 17:34:50 +08:00
|
|
|
BeginTs Timestamp
|
|
|
|
EndTs Timestamp
|
|
|
|
Msgs []TsMsg
|
|
|
|
StartPositions []*MsgPosition
|
2021-03-16 17:55:42 +08:00
|
|
|
EndPositions []*MsgPosition
|
2020-10-29 20:42:47 +08:00
|
|
|
}
|
|
|
|
|
2021-09-27 23:52:02 +08:00
|
|
|
// RepackFunc is a function type which used to repack message after hash by primary key
|
2020-11-17 14:10:07 +08:00
|
|
|
type RepackFunc func(msgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error)
|
2020-10-29 20:42:47 +08:00
|
|
|
|
2021-09-23 21:57:55 +08:00
|
|
|
// MsgStream is an interface that can be used to produce and consume message on message queue
|
2020-10-29 20:42:47 +08:00
|
|
|
type MsgStream interface {
|
2020-11-02 16:01:04 +08:00
|
|
|
Start()
|
|
|
|
Close()
|
2021-02-03 17:30:10 +08:00
|
|
|
Chan() <-chan *MsgPack
|
2021-02-04 14:37:12 +08:00
|
|
|
AsProducer(channels []string)
|
|
|
|
AsConsumer(channels []string, subName string)
|
2021-11-24 17:47:15 +08:00
|
|
|
AsReader(channels []string, subName string)
|
2021-10-15 11:46:33 +08:00
|
|
|
AsConsumerWithPosition(channels []string, subName string, position mqclient.SubscriptionInitialPosition)
|
2021-02-04 14:37:12 +08:00
|
|
|
SetRepackFunc(repackFunc RepackFunc)
|
2021-05-24 16:30:09 +08:00
|
|
|
ComputeProduceChannelIndexes(tsMsgs []TsMsg) [][]int32
|
2021-05-25 19:53:15 +08:00
|
|
|
GetProduceChannels() []string
|
2021-03-25 14:41:46 +08:00
|
|
|
Produce(*MsgPack) error
|
2021-10-11 11:38:57 +08:00
|
|
|
ProduceMark(*MsgPack) (map[string][]MessageID, error)
|
2021-03-25 14:41:46 +08:00
|
|
|
Broadcast(*MsgPack) error
|
2021-09-27 14:10:09 +08:00
|
|
|
BroadcastMark(*MsgPack) (map[string][]MessageID, error)
|
2021-03-25 14:41:46 +08:00
|
|
|
Consume() *MsgPack
|
2021-11-19 15:57:12 +08:00
|
|
|
Next(ctx context.Context, channelName string) (TsMsg, error)
|
2021-05-29 23:21:34 +08:00
|
|
|
Seek(offset []*MsgPosition) error
|
2021-11-19 15:57:12 +08:00
|
|
|
SeekReaders(msgPositions []*internalpb.MsgPosition) error
|
2020-11-12 11:18:23 +08:00
|
|
|
}
|
2021-02-04 14:37:12 +08:00
|
|
|
|
2021-09-23 21:57:55 +08:00
|
|
|
// Factory is an interface that can be used to generate a new msgstream object
|
2021-02-04 14:37:12 +08:00
|
|
|
type Factory interface {
|
2021-02-08 14:30:54 +08:00
|
|
|
SetParams(params map[string]interface{}) error
|
2021-02-04 14:37:12 +08:00
|
|
|
NewMsgStream(ctx context.Context) (MsgStream, error)
|
|
|
|
NewTtMsgStream(ctx context.Context) (MsgStream, error)
|
2021-03-19 20:16:04 +08:00
|
|
|
NewQueryMsgStream(ctx context.Context) (MsgStream, error)
|
2021-02-04 14:37:12 +08:00
|
|
|
}
|