mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-05 05:18:52 +08:00
a4439cc911
- Implement flusher to: - Manage the pipelines (creation, deletion, etc.) - Manage the segment write buffer - Manage sync operation (including receive flushMsg and execute flush) - Add a new `GetChannelRecoveryInfo` RPC in DataCoord. - Reorganize packages: `flushcommon` and `datanode`. issue: https://github.com/milvus-io/milvus/issues/33285 --------- Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
121 lines
3.5 KiB
Go
121 lines
3.5 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/milvus-io/milvus/internal/flushcommon/util"
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
"github.com/milvus-io/milvus/pkg/metrics"
|
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
|
)
|
|
|
|
type FlowgraphManager interface {
|
|
AddFlowgraph(ds *DataSyncService)
|
|
RemoveFlowgraph(channel string)
|
|
ClearFlowgraphs()
|
|
|
|
GetFlowgraphService(channel string) (*DataSyncService, bool)
|
|
HasFlowgraph(channel string) bool
|
|
HasFlowgraphWithOpID(channel string, opID int64) bool
|
|
GetFlowgraphCount() int
|
|
GetCollectionIDs() []int64
|
|
|
|
Close()
|
|
}
|
|
|
|
var _ FlowgraphManager = (*fgManagerImpl)(nil)
|
|
|
|
type fgManagerImpl struct {
|
|
ctx context.Context
|
|
cancelFunc context.CancelFunc
|
|
flowgraphs *typeutil.ConcurrentMap[string, *DataSyncService]
|
|
}
|
|
|
|
func NewFlowgraphManager() *fgManagerImpl {
|
|
ctx, cancelFunc := context.WithCancel(context.TODO())
|
|
return &fgManagerImpl{
|
|
ctx: ctx,
|
|
cancelFunc: cancelFunc,
|
|
flowgraphs: typeutil.NewConcurrentMap[string, *DataSyncService](),
|
|
}
|
|
}
|
|
|
|
func (fm *fgManagerImpl) AddFlowgraph(ds *DataSyncService) {
|
|
fm.flowgraphs.Insert(ds.vchannelName, ds)
|
|
metrics.DataNodeNumFlowGraphs.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Inc()
|
|
}
|
|
|
|
func (fm *fgManagerImpl) RemoveFlowgraph(channel string) {
|
|
if fg, loaded := fm.flowgraphs.Get(channel); loaded {
|
|
fg.close()
|
|
fm.flowgraphs.Remove(channel)
|
|
|
|
metrics.DataNodeNumFlowGraphs.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Dec()
|
|
util.GetRateCollector().RemoveFlowGraphChannel(channel)
|
|
}
|
|
}
|
|
|
|
func (fm *fgManagerImpl) ClearFlowgraphs() {
|
|
log.Info("start drop all flowgraph resources in DataNode")
|
|
fm.flowgraphs.Range(func(key string, value *DataSyncService) bool {
|
|
value.GracefullyClose()
|
|
fm.flowgraphs.GetAndRemove(key)
|
|
|
|
log.Info("successfully dropped flowgraph", zap.String("vChannelName", key))
|
|
return true
|
|
})
|
|
}
|
|
|
|
func (fm *fgManagerImpl) GetFlowgraphService(channel string) (*DataSyncService, bool) {
|
|
return fm.flowgraphs.Get(channel)
|
|
}
|
|
|
|
func (fm *fgManagerImpl) HasFlowgraph(channel string) bool {
|
|
_, exist := fm.flowgraphs.Get(channel)
|
|
return exist
|
|
}
|
|
|
|
func (fm *fgManagerImpl) HasFlowgraphWithOpID(channel string, opID typeutil.UniqueID) bool {
|
|
ds, exist := fm.flowgraphs.Get(channel)
|
|
return exist && ds.opID == opID
|
|
}
|
|
|
|
// GetFlowgraphCount returns number of flow graphs.
|
|
func (fm *fgManagerImpl) GetFlowgraphCount() int {
|
|
return fm.flowgraphs.Len()
|
|
}
|
|
|
|
func (fm *fgManagerImpl) GetCollectionIDs() []int64 {
|
|
collectionSet := typeutil.UniqueSet{}
|
|
fm.flowgraphs.Range(func(key string, value *DataSyncService) bool {
|
|
collectionSet.Insert(value.metacache.Collection())
|
|
return true
|
|
})
|
|
|
|
return collectionSet.Collect()
|
|
}
|
|
|
|
func (fm *fgManagerImpl) Close() {
|
|
fm.cancelFunc()
|
|
}
|