test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 10:59:32 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: add metrics for segment prune latnecy(#30376) (#34364)

Browse Source 
related: #30376
pr: https://github.com/milvus-io/milvus/pull/34094

Signed-off-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: MrPresent-Han <chun.han@gmail.com>
This commit is contained in:
Chun Han 2024-07-03 22:14:09 -04:00 committed by GitHub
parent 9087b6f42e
commit e12b701c03
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 24 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
internal/querynodev2/delegator/segment_pruner.go
View File

@ -24,6 +24,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/distance"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/timerecord"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -48,7 +49,7 @@ func PruneSegments(ctx context.Context,
// no need to prune
return
}
tr := timerecord.NewTimeRecorder("PruneSegments")
var collectionID int64
var expr []byte
if searchReq != nil {
@ -120,16 +121,23 @@ func PruneSegments(ctx context.Context,
item.Segments = newSegments
sealedSegments[idx] = item
}
filterRatio := float32(realFilteredSegments) / float32(totalSegNum)
metrics.QueryNodeSegmentPruneRatio.
WithLabelValues(fmt.Sprint(collectionID), fmt.Sprint(typeutil.IsVectorType(clusteringKeyField.GetDataType()))).
Observe(float64(realFilteredSegments / totalSegNum))
Observe(float64(filterRatio))
log.Ctx(ctx).Debug("Pruned segment for search/query",
zap.Int("filtered_segment_num[stats]", len(filteredSegments)),
zap.Int("filtered_segment_num[excluded]", realFilteredSegments),
zap.Int("total_segment_num", totalSegNum),
zap.Float32("filtered_ratio", float32(realFilteredSegments)/float32(totalSegNum)),
zap.Float32("filtered_ratio", filterRatio),
)
}
metrics.QueryNodeSegmentPruneLatency.WithLabelValues(fmt.Sprint(collectionID),
fmt.Sprint(typeutil.IsVectorType(clusteringKeyField.GetDataType()))).
Observe(float64(tr.ElapseSpan().Milliseconds()))
log.Ctx(ctx).Debug("Pruned segment for search/query",
zap.Duration("duration", tr.ElapseSpan()))
}
type segmentDisStruct struct {

13
pkg/metrics/querynode_metrics.go
View File

@ -374,6 +374,18 @@ var (
isVectorFieldLabelName,
})
QueryNodeSegmentPruneLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.QueryNodeRole,
Name: "segment_prune_latency",
Help: "latency of segment prune",
Buckets: buckets,
}, []string{
collectionIDLabelName,
isVectorFieldLabelName,
})
QueryNodeEvictedReadReqCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: milvusNamespace,
@ -789,6 +801,7 @@ func RegisterQueryNode(registry *prometheus.Registry) {
registry.MustRegister(QueryNodeDiskCacheEvictBytes)
registry.MustRegister(QueryNodeDiskCacheEvictDuration)
registry.MustRegister(QueryNodeDiskCacheEvictGlobalDuration)
registry.MustRegister(QueryNodeSegmentPruneLatency)
registry.MustRegister(QueryNodeApplyBFCost)
registry.MustRegister(QueryNodeForwardDeleteCost)
registry.MustRegister(QueryNodeSegmentPruneRatio)