test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 11:59:00 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Add receivedNQ metric and change receivedBytes metric to collection level (#24199)

Browse Source 
Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
yihao.dai 2023-05-21 22:23:24 +08:00 committed by GitHub
parent 0b72cf2c2e
commit 3f96c335bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 47 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
internal/proxy/impl.go
View File

@ -2115,7 +2115,9 @@ func (node *Proxy) Insert(ctx context.Context, request *milvuspb.InsertRequest)
}
method := "Insert"
tr := timerecord.NewTimeRecorder(method)
metrics.ProxyReceiveBytes.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.InsertLabel).Add(float64(proto.Size(request)))
metrics.ProxyReceiveBytes.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.InsertLabel, request.GetCollectionName()).Add(float64(proto.Size(request)))
metrics.ProxyFunctionCall.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method, metrics.TotalLabel).Inc()
it := &insertTask{
@ -2232,7 +2234,9 @@ func (node *Proxy) Delete(ctx context.Context, request *milvuspb.DeleteRequest)
log.Debug("Start processing delete request in Proxy")
defer log.Debug("Finish processing delete request in Proxy")
metrics.ProxyReceiveBytes.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.DeleteLabel).Add(float64(proto.Size(request)))
metrics.ProxyReceiveBytes.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.DeleteLabel, request.GetCollectionName()).Add(float64(proto.Size(request)))
if !node.checkHealthy() {
return &milvuspb.MutationResult{
@ -2342,7 +2346,9 @@ func (node *Proxy) Upsert(ctx context.Context, request *milvuspb.UpsertRequest)
method := "Upsert"
tr := timerecord.NewTimeRecorder(method)
metrics.ProxyReceiveBytes.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.UpsertLabel).Add(float64(proto.Size(request)))
metrics.ProxyReceiveBytes.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.UpsertLabel, request.GetCollectionName()).Add(float64(proto.Size(request)))
metrics.ProxyFunctionCall.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method, metrics.TotalLabel).Inc()
it := &upsertTask{
@ -2462,7 +2468,13 @@ func (node *Proxy) Upsert(ctx context.Context, request *milvuspb.UpsertRequest)
// Search search the most similar records of requests.
func (node *Proxy) Search(ctx context.Context, request *milvuspb.SearchRequest) (*milvuspb.SearchResults, error) {
receiveSize := proto.Size(request)
metrics.ProxyReceiveBytes.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.SearchLabel).Add(float64(receiveSize))
metrics.ProxyReceiveBytes.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.SearchLabel, request.GetCollectionName()).Add(float64(receiveSize))
metrics.ProxyReceivedNQ.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.SearchLabel, request.GetCollectionName()).Add(float64(request.GetNq()))
rateCol.Add(internalpb.RateType_DQLSearch.String(), float64(request.GetNq()))
@ -2657,7 +2669,13 @@ func (node *Proxy) Flush(ctx context.Context, request *milvuspb.FlushRequest) (*
// Query get the records by primary keys.
func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (*milvuspb.QueryResults, error) {
receiveSize := proto.Size(request)
metrics.ProxyReceiveBytes.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.QueryLabel).Add(float64(receiveSize))
metrics.ProxyReceiveBytes.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.QueryLabel, request.GetCollectionName()).Add(float64(receiveSize))
metrics.ProxyReceivedNQ.WithLabelValues(
strconv.FormatInt(paramtable.GetNodeID(), 10),
metrics.SearchLabel, request.GetCollectionName()).Add(float64(1))
rateCol.Add(internalpb.RateType_DQLQuery.String(), 1)

25
pkg/metrics/proxy_metrics.go
View File

@ -25,6 +25,14 @@ import (
)
var (
ProxyReceivedNQ = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.ProxyRole,
Name: "received_nq",
Help: "counter of nq of received search and query requests",
}, []string{nodeIDLabelName, queryTypeLabelName, collectionName})
// ProxySearchVectors record the number of vectors search successfully.
ProxySearchVectors = prometheus.NewCounterVec(
prometheus.CounterOpts{
@ -204,7 +212,7 @@ var (
Subsystem: typeutil.ProxyRole,
Name: "receive_bytes_count",
Help: "count of bytes received from sdk",
}, []string{nodeIDLabelName, msgTypeLabelName})
}, []string{nodeIDLabelName, msgTypeLabelName, collectionName})
// ProxyReadReqSendBytes record the bytes sent back to client by Proxy
ProxyReadReqSendBytes = prometheus.NewCounterVec(
@ -243,6 +251,7 @@ var (
// RegisterProxy registers Proxy metrics
func RegisterProxy(registry *prometheus.Registry) {
registry.MustRegister(ProxyReceivedNQ)
registry.MustRegister(ProxySearchVectors)
registry.MustRegister(ProxyInsertVectors)
@ -286,4 +295,18 @@ func CleanupCollectionMetrics(nodeID int64, collection string) {
msgTypeLabelName: InsertLabel, collectionName: collection})
ProxyCollectionMutationLatency.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
msgTypeLabelName: DeleteLabel, collectionName: collection})
ProxyReceivedNQ.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
queryTypeLabelName: SearchLabel, collectionName: collection})
ProxyReceivedNQ.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
queryTypeLabelName: QueryLabel, collectionName: collection})
ProxyReceiveBytes.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
msgTypeLabelName: SearchLabel, collectionName: collection})
ProxyReceiveBytes.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
msgTypeLabelName: QueryLabel, collectionName: collection})
ProxyReceiveBytes.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
msgTypeLabelName: InsertLabel, collectionName: collection})
ProxyReceiveBytes.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
msgTypeLabelName: DeleteLabel, collectionName: collection})
ProxyReceiveBytes.Delete(prometheus.Labels{nodeIDLabelName: strconv.FormatInt(nodeID, 10),
msgTypeLabelName: UpsertLabel, collectionName: collection})
}