2022-01-13 14:21:34 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2024-07-01 14:46:07 +08:00
|
|
|
package pipeline
|
2022-01-13 14:21:34 +08:00
|
|
|
|
|
|
|
import (
|
2023-09-20 16:03:23 +08:00
|
|
|
"context"
|
2022-01-13 14:21:34 +08:00
|
|
|
"fmt"
|
2023-03-21 21:37:56 +08:00
|
|
|
|
|
|
|
"go.uber.org/zap"
|
2022-01-13 14:21:34 +08:00
|
|
|
|
2024-08-02 18:30:23 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/flushcommon/util"
|
2023-04-06 19:14:32 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
|
|
"github.com/milvus-io/milvus/pkg/metrics"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
2023-07-24 10:21:07 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
2022-01-13 14:21:34 +08:00
|
|
|
)
|
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
type FlowgraphManager interface {
|
2024-07-01 14:46:07 +08:00
|
|
|
AddFlowgraph(ds *DataSyncService)
|
2023-11-30 18:42:32 +08:00
|
|
|
RemoveFlowgraph(channel string)
|
|
|
|
ClearFlowgraphs()
|
|
|
|
|
2024-07-01 14:46:07 +08:00
|
|
|
GetFlowgraphService(channel string) (*DataSyncService, bool)
|
2023-11-30 18:42:32 +08:00
|
|
|
HasFlowgraph(channel string) bool
|
2024-07-01 14:46:07 +08:00
|
|
|
HasFlowgraphWithOpID(channel string, opID int64) bool
|
2023-11-30 18:42:32 +08:00
|
|
|
GetFlowgraphCount() int
|
|
|
|
GetCollectionIDs() []int64
|
2024-04-16 10:49:20 +08:00
|
|
|
|
|
|
|
Close()
|
2023-03-21 21:37:56 +08:00
|
|
|
}
|
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
var _ FlowgraphManager = (*fgManagerImpl)(nil)
|
2023-03-21 21:37:56 +08:00
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
type fgManagerImpl struct {
|
2024-04-16 10:49:20 +08:00
|
|
|
ctx context.Context
|
|
|
|
cancelFunc context.CancelFunc
|
2024-07-01 14:46:07 +08:00
|
|
|
flowgraphs *typeutil.ConcurrentMap[string, *DataSyncService]
|
2023-03-21 21:37:56 +08:00
|
|
|
}
|
|
|
|
|
2024-07-01 14:46:07 +08:00
|
|
|
func NewFlowgraphManager() *fgManagerImpl {
|
2024-04-16 10:49:20 +08:00
|
|
|
ctx, cancelFunc := context.WithCancel(context.TODO())
|
2023-11-30 18:42:32 +08:00
|
|
|
return &fgManagerImpl{
|
2024-04-16 10:49:20 +08:00
|
|
|
ctx: ctx,
|
|
|
|
cancelFunc: cancelFunc,
|
2024-07-01 14:46:07 +08:00
|
|
|
flowgraphs: typeutil.NewConcurrentMap[string, *DataSyncService](),
|
2023-03-21 21:37:56 +08:00
|
|
|
}
|
2022-01-13 14:21:34 +08:00
|
|
|
}
|
|
|
|
|
2024-07-01 14:46:07 +08:00
|
|
|
func (fm *fgManagerImpl) AddFlowgraph(ds *DataSyncService) {
|
2023-10-08 21:37:33 +08:00
|
|
|
fm.flowgraphs.Insert(ds.vchannelName, ds)
|
|
|
|
metrics.DataNodeNumFlowGraphs.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Inc()
|
|
|
|
}
|
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
func (fm *fgManagerImpl) RemoveFlowgraph(channel string) {
|
|
|
|
if fg, loaded := fm.flowgraphs.Get(channel); loaded {
|
2023-07-24 10:21:07 +08:00
|
|
|
fg.close()
|
2023-11-30 18:42:32 +08:00
|
|
|
fm.flowgraphs.Remove(channel)
|
2023-09-19 11:53:22 +08:00
|
|
|
|
2022-11-04 14:25:38 +08:00
|
|
|
metrics.DataNodeNumFlowGraphs.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Dec()
|
2024-08-02 18:30:23 +08:00
|
|
|
util.GetRateCollector().RemoveFlowGraphChannel(channel)
|
2022-01-13 14:21:34 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
func (fm *fgManagerImpl) ClearFlowgraphs() {
|
|
|
|
log.Info("start drop all flowgraph resources in DataNode")
|
2024-07-01 14:46:07 +08:00
|
|
|
fm.flowgraphs.Range(func(key string, value *DataSyncService) bool {
|
2023-11-30 18:42:32 +08:00
|
|
|
value.GracefullyClose()
|
|
|
|
fm.flowgraphs.GetAndRemove(key)
|
|
|
|
|
|
|
|
log.Info("successfully dropped flowgraph", zap.String("vChannelName", key))
|
|
|
|
return true
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2024-07-01 14:46:07 +08:00
|
|
|
func (fm *fgManagerImpl) GetFlowgraphService(channel string) (*DataSyncService, bool) {
|
2023-11-30 18:42:32 +08:00
|
|
|
return fm.flowgraphs.Get(channel)
|
2022-01-13 14:21:34 +08:00
|
|
|
}
|
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
func (fm *fgManagerImpl) HasFlowgraph(channel string) bool {
|
|
|
|
_, exist := fm.flowgraphs.Get(channel)
|
2022-01-13 14:21:34 +08:00
|
|
|
return exist
|
|
|
|
}
|
|
|
|
|
2024-08-02 18:30:23 +08:00
|
|
|
func (fm *fgManagerImpl) HasFlowgraphWithOpID(channel string, opID typeutil.UniqueID) bool {
|
2023-11-30 18:42:32 +08:00
|
|
|
ds, exist := fm.flowgraphs.Get(channel)
|
2023-10-08 21:37:33 +08:00
|
|
|
return exist && ds.opID == opID
|
|
|
|
}
|
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
// GetFlowgraphCount returns number of flow graphs.
|
|
|
|
func (fm *fgManagerImpl) GetFlowgraphCount() int {
|
2023-07-24 10:21:07 +08:00
|
|
|
return fm.flowgraphs.Len()
|
2022-09-16 09:56:47 +08:00
|
|
|
}
|
|
|
|
|
2023-11-30 18:42:32 +08:00
|
|
|
func (fm *fgManagerImpl) GetCollectionIDs() []int64 {
|
2023-09-19 11:53:22 +08:00
|
|
|
collectionSet := typeutil.UniqueSet{}
|
2024-07-01 14:46:07 +08:00
|
|
|
fm.flowgraphs.Range(func(key string, value *DataSyncService) bool {
|
2023-11-15 15:24:18 +08:00
|
|
|
collectionSet.Insert(value.metacache.Collection())
|
2023-09-19 11:53:22 +08:00
|
|
|
return true
|
|
|
|
})
|
|
|
|
|
|
|
|
return collectionSet.Collect()
|
|
|
|
}
|
2024-04-16 10:49:20 +08:00
|
|
|
|
|
|
|
func (fm *fgManagerImpl) Close() {
|
|
|
|
fm.cancelFunc()
|
|
|
|
}
|