2021-11-29 20:13:50 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
2021-04-19 13:47:10 +08:00
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
2021-11-29 20:13:50 +08:00
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
2021-04-19 13:47:10 +08:00
|
|
|
//
|
2021-11-29 20:13:50 +08:00
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2021-04-19 13:47:10 +08:00
|
|
|
|
2021-01-16 10:12:14 +08:00
|
|
|
package querynode
|
2020-11-09 16:27:11 +08:00
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2021-06-09 11:37:55 +08:00
|
|
|
"fmt"
|
|
|
|
"sync"
|
2020-11-09 16:27:11 +08:00
|
|
|
|
2021-03-05 09:21:35 +08:00
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/log"
|
|
|
|
"github.com/milvus-io/milvus/internal/msgstream"
|
2020-11-09 16:27:11 +08:00
|
|
|
)
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// dataSyncService manages a lot of flow graphs
|
2021-06-09 11:37:55 +08:00
|
|
|
type dataSyncService struct {
|
|
|
|
ctx context.Context
|
2021-01-16 10:12:14 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
mu sync.Mutex // guards FlowGraphs
|
|
|
|
dmlChannel2FlowGraph map[Channel]*queryNodeFlowGraph
|
|
|
|
deltaChannel2FlowGraph map[Channel]*queryNodeFlowGraph
|
2021-06-09 11:37:55 +08:00
|
|
|
|
2021-10-26 14:46:19 +08:00
|
|
|
streamingReplica ReplicaInterface
|
|
|
|
historicalReplica ReplicaInterface
|
|
|
|
tSafeReplica TSafeReplicaInterface
|
|
|
|
msFactory msgstream.Factory
|
2020-11-09 16:27:11 +08:00
|
|
|
}
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// addFlowGraphsForDMLChannels add flowGraphs to dmlChannel2FlowGraph
|
|
|
|
func (dsService *dataSyncService) addFlowGraphsForDMLChannels(collectionID UniqueID, dmlChannels []string) {
|
2021-06-09 11:37:55 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
2021-01-26 09:38:40 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
for _, channel := range dmlChannels {
|
|
|
|
if _, ok := dsService.dmlChannel2FlowGraph[channel]; ok {
|
|
|
|
log.Warn("dml flow graph has been existed",
|
|
|
|
zap.Any("collectionID", collectionID),
|
|
|
|
zap.Any("channel", channel),
|
|
|
|
)
|
|
|
|
continue
|
|
|
|
}
|
2021-06-09 11:37:55 +08:00
|
|
|
newFlowGraph := newQueryNodeFlowGraph(dsService.ctx,
|
|
|
|
collectionID,
|
|
|
|
dsService.streamingReplica,
|
|
|
|
dsService.tSafeReplica,
|
2021-12-17 20:12:42 +08:00
|
|
|
channel,
|
2021-06-09 11:37:55 +08:00
|
|
|
dsService.msFactory)
|
2021-12-17 20:12:42 +08:00
|
|
|
dsService.dmlChannel2FlowGraph[channel] = newFlowGraph
|
|
|
|
log.Debug("add DML flow graph",
|
2021-06-09 11:37:55 +08:00
|
|
|
zap.Any("collectionID", collectionID),
|
2021-12-17 20:12:42 +08:00
|
|
|
zap.Any("channel", channel))
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
2020-11-09 16:27:11 +08:00
|
|
|
}
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// addFlowGraphsForDeltaChannels add flowGraphs to deltaChannel2FlowGraph
|
|
|
|
func (dsService *dataSyncService) addFlowGraphsForDeltaChannels(collectionID UniqueID, deltaChannels []string) {
|
2021-11-06 11:02:58 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
for _, channel := range deltaChannels {
|
|
|
|
if _, ok := dsService.deltaChannel2FlowGraph[channel]; ok {
|
|
|
|
log.Warn("delta flow graph has been existed",
|
|
|
|
zap.Any("collectionID", collectionID),
|
|
|
|
zap.Any("channel", channel),
|
|
|
|
)
|
|
|
|
continue
|
|
|
|
}
|
2021-11-06 11:02:58 +08:00
|
|
|
newFlowGraph := newQueryNodeDeltaFlowGraph(dsService.ctx,
|
|
|
|
collectionID,
|
|
|
|
dsService.historicalReplica,
|
|
|
|
dsService.tSafeReplica,
|
2021-12-17 20:12:42 +08:00
|
|
|
channel,
|
2021-11-06 11:02:58 +08:00
|
|
|
dsService.msFactory)
|
2021-12-17 20:12:42 +08:00
|
|
|
dsService.deltaChannel2FlowGraph[channel] = newFlowGraph
|
|
|
|
log.Debug("add delta flow graph",
|
2021-11-06 11:02:58 +08:00
|
|
|
zap.Any("collectionID", collectionID),
|
2021-12-17 20:12:42 +08:00
|
|
|
zap.Any("channel", channel))
|
2021-11-06 11:02:58 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// getFlowGraphByDMLChannel returns the DML flowGraph by channel
|
|
|
|
func (dsService *dataSyncService) getFlowGraphByDMLChannel(collectionID UniqueID, channel Channel) (*queryNodeFlowGraph, error) {
|
2021-06-09 11:37:55 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
2020-11-09 16:27:11 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
if _, ok := dsService.dmlChannel2FlowGraph[channel]; !ok {
|
|
|
|
return nil, fmt.Errorf("DML flow graph doesn't existed, collectionID = %d", collectionID)
|
2020-11-24 16:12:39 +08:00
|
|
|
}
|
2021-06-15 12:41:40 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// TODO: return clone?
|
|
|
|
return dsService.dmlChannel2FlowGraph[channel], nil
|
2020-11-12 12:04:12 +08:00
|
|
|
}
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// getFlowGraphByDeltaChannel returns the delta flowGraph by channel
|
|
|
|
func (dsService *dataSyncService) getFlowGraphByDeltaChannel(collectionID UniqueID, channel Channel) (*queryNodeFlowGraph, error) {
|
2021-11-06 11:02:58 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
if _, ok := dsService.deltaChannel2FlowGraph[channel]; !ok {
|
|
|
|
return nil, fmt.Errorf("delta flow graph doesn't existed, collectionID = %d", collectionID)
|
2021-11-06 11:02:58 +08:00
|
|
|
}
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// TODO: return clone?
|
|
|
|
return dsService.deltaChannel2FlowGraph[channel], nil
|
2021-11-06 11:02:58 +08:00
|
|
|
}
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// startFlowGraphByDMLChannel starts the DML flow graph by channel
|
|
|
|
func (dsService *dataSyncService) startFlowGraphByDMLChannel(collectionID UniqueID, channel Channel) error {
|
2021-06-09 11:37:55 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
2020-11-09 16:27:11 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
if _, ok := dsService.dmlChannel2FlowGraph[channel]; !ok {
|
|
|
|
return fmt.Errorf("DML flow graph doesn't existed, collectionID = %d", collectionID)
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
2021-12-17 20:12:42 +08:00
|
|
|
log.Debug("start DML flow graph",
|
|
|
|
zap.Any("collectionID", collectionID),
|
|
|
|
zap.Any("channel", channel),
|
|
|
|
)
|
|
|
|
dsService.dmlChannel2FlowGraph[channel].flowGraph.Start()
|
2021-06-09 11:37:55 +08:00
|
|
|
return nil
|
|
|
|
}
|
2020-12-10 16:31:09 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// startFlowGraphForDeltaChannel would start the delta flow graph by channel
|
|
|
|
func (dsService *dataSyncService) startFlowGraphForDeltaChannel(collectionID UniqueID, channel Channel) error {
|
2021-11-06 11:02:58 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
if _, ok := dsService.deltaChannel2FlowGraph[channel]; !ok {
|
|
|
|
return fmt.Errorf("delta flow graph doesn't existed, collectionID = %d", collectionID)
|
2021-11-06 11:02:58 +08:00
|
|
|
}
|
2021-12-17 20:12:42 +08:00
|
|
|
log.Debug("start delta flow graph",
|
|
|
|
zap.Any("collectionID", collectionID),
|
|
|
|
zap.Any("channel", channel),
|
|
|
|
)
|
|
|
|
dsService.deltaChannel2FlowGraph[channel].flowGraph.Start()
|
2021-11-06 11:02:58 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// removeFlowGraphsByDMLChannels would remove the DML flow graphs by channels
|
|
|
|
func (dsService *dataSyncService) removeFlowGraphsByDMLChannels(channels []Channel) {
|
2021-06-09 11:37:55 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
2020-12-10 16:31:09 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
for _, channel := range channels {
|
|
|
|
if _, ok := dsService.dmlChannel2FlowGraph[channel]; ok {
|
2021-06-09 11:37:55 +08:00
|
|
|
// close flow graph
|
2021-12-17 20:12:42 +08:00
|
|
|
dsService.dmlChannel2FlowGraph[channel].close()
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
2021-12-17 20:12:42 +08:00
|
|
|
delete(dsService.dmlChannel2FlowGraph, channel)
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
|
|
|
}
|
2020-11-09 16:27:11 +08:00
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
// removeFlowGraphsByDeltaChannels would remove the delta flow graphs by channels
|
|
|
|
func (dsService *dataSyncService) removeFlowGraphsByDeltaChannels(channels []Channel) {
|
2021-11-06 11:02:58 +08:00
|
|
|
dsService.mu.Lock()
|
|
|
|
defer dsService.mu.Unlock()
|
|
|
|
|
2021-12-17 20:12:42 +08:00
|
|
|
for _, channel := range channels {
|
|
|
|
if _, ok := dsService.deltaChannel2FlowGraph[channel]; ok {
|
2021-11-06 11:02:58 +08:00
|
|
|
// close flow graph
|
2021-12-17 20:12:42 +08:00
|
|
|
dsService.deltaChannel2FlowGraph[channel].close()
|
2021-06-15 12:41:40 +08:00
|
|
|
}
|
2021-12-17 20:12:42 +08:00
|
|
|
delete(dsService.deltaChannel2FlowGraph, channel)
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
|
|
|
}
|
2020-11-09 16:27:11 +08:00
|
|
|
|
2021-10-01 09:53:43 +08:00
|
|
|
// newDataSyncService returns a new dataSyncService
|
2021-06-09 11:37:55 +08:00
|
|
|
func newDataSyncService(ctx context.Context,
|
|
|
|
streamingReplica ReplicaInterface,
|
2021-10-26 14:46:19 +08:00
|
|
|
historicalReplica ReplicaInterface,
|
2021-06-09 11:37:55 +08:00
|
|
|
tSafeReplica TSafeReplicaInterface,
|
|
|
|
factory msgstream.Factory) *dataSyncService {
|
|
|
|
|
|
|
|
return &dataSyncService{
|
2021-12-17 20:12:42 +08:00
|
|
|
ctx: ctx,
|
|
|
|
dmlChannel2FlowGraph: make(map[Channel]*queryNodeFlowGraph),
|
|
|
|
deltaChannel2FlowGraph: make(map[Channel]*queryNodeFlowGraph),
|
|
|
|
streamingReplica: streamingReplica,
|
|
|
|
historicalReplica: historicalReplica,
|
|
|
|
tSafeReplica: tSafeReplica,
|
|
|
|
msFactory: factory,
|
2020-11-09 16:27:11 +08:00
|
|
|
}
|
|
|
|
}
|
2021-03-04 14:49:51 +08:00
|
|
|
|
2021-11-18 09:25:17 +08:00
|
|
|
// close would close and remove all flow graphs in dataSyncService
|
2021-06-09 11:37:55 +08:00
|
|
|
func (dsService *dataSyncService) close() {
|
2021-12-17 20:12:42 +08:00
|
|
|
// close DML flow graphs
|
|
|
|
for channel, nodeFG := range dsService.dmlChannel2FlowGraph {
|
|
|
|
if nodeFG != nil {
|
|
|
|
nodeFG.flowGraph.Close()
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
2021-12-17 20:12:42 +08:00
|
|
|
delete(dsService.dmlChannel2FlowGraph, channel)
|
2021-03-04 14:49:51 +08:00
|
|
|
}
|
2021-12-17 20:12:42 +08:00
|
|
|
// close delta flow graphs
|
|
|
|
for channel, nodeFG := range dsService.deltaChannel2FlowGraph {
|
|
|
|
if nodeFG != nil {
|
|
|
|
nodeFG.flowGraph.Close()
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
2021-12-17 20:12:42 +08:00
|
|
|
delete(dsService.deltaChannel2FlowGraph, channel)
|
2021-06-09 11:37:55 +08:00
|
|
|
}
|
2021-03-04 14:49:51 +08:00
|
|
|
}
|