mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-11-30 10:59:32 +08:00
267 lines
8.9 KiB
Go
267 lines
8.9 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package querynodev2
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/samber/lo"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
|
|
"github.com/milvus-io/milvus/internal/querynodev2/collector"
|
|
"github.com/milvus-io/milvus/internal/querynodev2/segments"
|
|
"github.com/milvus-io/milvus/pkg/metrics"
|
|
"github.com/milvus-io/milvus/pkg/util/hardware"
|
|
"github.com/milvus-io/milvus/pkg/util/merr"
|
|
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
|
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
|
"github.com/milvus-io/milvus/pkg/util/ratelimitutil"
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
|
)
|
|
|
|
func getRateMetric() ([]metricsinfo.RateMetric, error) {
|
|
rms := make([]metricsinfo.RateMetric, 0)
|
|
for _, label := range collector.RateMetrics() {
|
|
rate, err := collector.Rate.Rate(label, ratelimitutil.DefaultAvgDuration)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
rms = append(rms, metricsinfo.RateMetric{
|
|
Label: label,
|
|
Rate: rate,
|
|
})
|
|
}
|
|
return rms, nil
|
|
}
|
|
|
|
func getSearchNQInQueue() (metricsinfo.ReadInfoInQueue, error) {
|
|
average, err := collector.Average.Average(metricsinfo.SearchQueueMetric)
|
|
if err != nil {
|
|
return metricsinfo.ReadInfoInQueue{}, err
|
|
}
|
|
defer collector.Average.Reset(metricsinfo.SearchQueueMetric)
|
|
|
|
readyQueueLabel := collector.ConstructLabel(metricsinfo.ReadyQueueType, metricsinfo.SearchQueueMetric)
|
|
executeQueueLabel := collector.ConstructLabel(metricsinfo.ExecuteQueueType, metricsinfo.SearchQueueMetric)
|
|
|
|
return metricsinfo.ReadInfoInQueue{
|
|
ReadyQueue: collector.Counter.Get(readyQueueLabel),
|
|
ExecuteChan: collector.Counter.Get(executeQueueLabel),
|
|
AvgQueueDuration: time.Duration(int64(average)),
|
|
}, nil
|
|
}
|
|
|
|
func getQueryTasksInQueue() (metricsinfo.ReadInfoInQueue, error) {
|
|
average, err := collector.Average.Average(metricsinfo.QueryQueueMetric)
|
|
if err != nil {
|
|
return metricsinfo.ReadInfoInQueue{}, err
|
|
}
|
|
defer collector.Average.Reset(metricsinfo.QueryQueueMetric)
|
|
|
|
readyQueueLabel := collector.ConstructLabel(metricsinfo.ReadyQueueType, metricsinfo.QueryQueueMetric)
|
|
executeQueueLabel := collector.ConstructLabel(metricsinfo.ExecuteQueueType, metricsinfo.QueryQueueMetric)
|
|
|
|
return metricsinfo.ReadInfoInQueue{
|
|
ReadyQueue: collector.Counter.Get(readyQueueLabel),
|
|
ExecuteChan: collector.Counter.Get(executeQueueLabel),
|
|
AvgQueueDuration: time.Duration(int64(average)),
|
|
}, nil
|
|
}
|
|
|
|
// getQuotaMetrics returns QueryNodeQuotaMetrics.
|
|
func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error) {
|
|
rms, err := getRateMetric()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sqms, err := getSearchNQInQueue()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
qqms, err := getQueryTasksInQueue()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
minTsafeChannel, minTsafe := node.tSafeManager.Min()
|
|
collections := node.manager.Collection.List()
|
|
nodeID := fmt.Sprint(node.GetNodeID())
|
|
|
|
metrics.QueryNodeNumEntities.Reset()
|
|
metrics.QueryNodeEntitiesSize.Reset()
|
|
|
|
var totalGrowingSize int64
|
|
growingSegments := node.manager.Segment.GetBy(segments.WithType(segments.SegmentTypeGrowing))
|
|
growingGroupByCollection := lo.GroupBy(growingSegments, func(seg segments.Segment) int64 {
|
|
return seg.Collection()
|
|
})
|
|
for _, collection := range collections {
|
|
segs := growingGroupByCollection[collection]
|
|
size := lo.SumBy(segs, func(seg segments.Segment) int64 {
|
|
return seg.MemSize()
|
|
})
|
|
totalGrowingSize += size
|
|
metrics.QueryNodeEntitiesSize.WithLabelValues(nodeID, fmt.Sprint(collection),
|
|
segments.SegmentTypeGrowing.String()).Set(float64(size))
|
|
}
|
|
growingGroupByPartition := lo.GroupBy(growingSegments, func(seg segments.Segment) int64 {
|
|
return seg.Partition()
|
|
})
|
|
|
|
for _, segs := range growingGroupByPartition {
|
|
numEntities := lo.SumBy(segs, func(seg segments.Segment) int64 {
|
|
return seg.RowNum()
|
|
})
|
|
segment := segs[0]
|
|
metrics.QueryNodeNumEntities.WithLabelValues(
|
|
segment.DatabaseName(),
|
|
nodeID,
|
|
fmt.Sprint(segment.Collection()),
|
|
fmt.Sprint(segment.Partition()),
|
|
segments.SegmentTypeGrowing.String(),
|
|
).Set(float64(numEntities))
|
|
}
|
|
|
|
sealedSegments := node.manager.Segment.GetBy(segments.WithType(segments.SegmentTypeSealed))
|
|
sealedGroupByCollection := lo.GroupBy(sealedSegments, func(seg segments.Segment) int64 {
|
|
return seg.Collection()
|
|
})
|
|
for _, collection := range collections {
|
|
segs := sealedGroupByCollection[collection]
|
|
size := lo.SumBy(segs, func(seg segments.Segment) int64 {
|
|
return seg.MemSize()
|
|
})
|
|
metrics.QueryNodeEntitiesSize.WithLabelValues(fmt.Sprint(node.GetNodeID()),
|
|
fmt.Sprint(collection), segments.SegmentTypeSealed.String()).Set(float64(size))
|
|
}
|
|
sealedGroupByPartition := lo.GroupBy(sealedSegments, func(seg segments.Segment) int64 {
|
|
return seg.Partition()
|
|
})
|
|
for _, segs := range sealedGroupByPartition {
|
|
numEntities := lo.SumBy(segs, func(seg segments.Segment) int64 {
|
|
return seg.RowNum()
|
|
})
|
|
segment := segs[0]
|
|
metrics.QueryNodeNumEntities.WithLabelValues(
|
|
segment.DatabaseName(),
|
|
nodeID,
|
|
fmt.Sprint(segment.Collection()),
|
|
fmt.Sprint(segment.Partition()),
|
|
segments.SegmentTypeSealed.String(),
|
|
).Set(float64(numEntities))
|
|
}
|
|
|
|
return &metricsinfo.QueryNodeQuotaMetrics{
|
|
Hms: metricsinfo.HardwareMetrics{},
|
|
Rms: rms,
|
|
Fgm: metricsinfo.FlowGraphMetric{
|
|
MinFlowGraphChannel: minTsafeChannel,
|
|
MinFlowGraphTt: minTsafe,
|
|
NumFlowGraph: node.pipelineManager.Num(),
|
|
},
|
|
SearchQueue: sqms,
|
|
QueryQueue: qqms,
|
|
GrowingSegmentsSize: totalGrowingSize,
|
|
Effect: metricsinfo.NodeEffect{
|
|
NodeID: node.GetNodeID(),
|
|
CollectionIDs: collections,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func getCollectionMetrics(node *QueryNode) (*metricsinfo.QueryNodeCollectionMetrics, error) {
|
|
allSegments := node.manager.Segment.GetBy()
|
|
ret := &metricsinfo.QueryNodeCollectionMetrics{
|
|
CollectionRows: make(map[int64]int64),
|
|
}
|
|
for _, segment := range allSegments {
|
|
collectionID := segment.Collection()
|
|
ret.CollectionRows[collectionID] += segment.RowNum()
|
|
}
|
|
return ret, nil
|
|
}
|
|
|
|
// getSystemInfoMetrics returns metrics info of QueryNode
|
|
func getSystemInfoMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest, node *QueryNode) (*milvuspb.GetMetricsResponse, error) {
|
|
usedMem := hardware.GetUsedMemoryCount()
|
|
totalMem := hardware.GetMemoryCount()
|
|
|
|
quotaMetrics, err := getQuotaMetrics(node)
|
|
if err != nil {
|
|
return &milvuspb.GetMetricsResponse{
|
|
Status: merr.Status(err),
|
|
ComponentName: metricsinfo.ConstructComponentName(typeutil.QueryNodeRole, node.GetNodeID()),
|
|
}, nil
|
|
}
|
|
hardwareInfos := metricsinfo.HardwareMetrics{
|
|
IP: node.session.Address,
|
|
CPUCoreCount: hardware.GetCPUNum(),
|
|
CPUCoreUsage: hardware.GetCPUUsage(),
|
|
Memory: totalMem,
|
|
MemoryUsage: usedMem,
|
|
Disk: hardware.GetDiskCount(),
|
|
DiskUsage: hardware.GetDiskUsage(),
|
|
}
|
|
quotaMetrics.Hms = hardwareInfos
|
|
|
|
collectionMetrics, err := getCollectionMetrics(node)
|
|
if err != nil {
|
|
return &milvuspb.GetMetricsResponse{
|
|
Status: merr.Status(err),
|
|
ComponentName: metricsinfo.ConstructComponentName(typeutil.QueryNodeRole, node.GetNodeID()),
|
|
}, nil
|
|
}
|
|
|
|
nodeInfos := metricsinfo.QueryNodeInfos{
|
|
BaseComponentInfos: metricsinfo.BaseComponentInfos{
|
|
Name: metricsinfo.ConstructComponentName(typeutil.QueryNodeRole, node.GetNodeID()),
|
|
HardwareInfos: hardwareInfos,
|
|
SystemInfo: metricsinfo.DeployMetrics{},
|
|
CreatedTime: paramtable.GetCreateTime().String(),
|
|
UpdatedTime: paramtable.GetUpdateTime().String(),
|
|
Type: typeutil.QueryNodeRole,
|
|
ID: node.session.ServerID,
|
|
},
|
|
SystemConfigurations: metricsinfo.QueryNodeConfiguration{
|
|
SimdType: paramtable.Get().CommonCfg.SimdType.GetValue(),
|
|
},
|
|
QuotaMetrics: quotaMetrics,
|
|
CollectionMetrics: collectionMetrics,
|
|
}
|
|
metricsinfo.FillDeployMetricsWithEnv(&nodeInfos.SystemInfo)
|
|
|
|
resp, err := metricsinfo.MarshalComponentInfos(nodeInfos)
|
|
if err != nil {
|
|
return &milvuspb.GetMetricsResponse{
|
|
Status: merr.Status(err),
|
|
Response: "",
|
|
ComponentName: metricsinfo.ConstructComponentName(typeutil.QueryNodeRole, node.GetNodeID()),
|
|
}, nil
|
|
}
|
|
|
|
return &milvuspb.GetMetricsResponse{
|
|
Status: merr.Success(),
|
|
Response: resp,
|
|
ComponentName: metricsinfo.ConstructComponentName(typeutil.QueryNodeRole, node.GetNodeID()),
|
|
}, nil
|
|
}
|