mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-01 03:18:29 +08:00
10356 lines
285 KiB
JSON
10356 lines
285 KiB
JSON
{
|
|
"__inputs": [
|
|
{
|
|
"name": "DS_PROMETHEUS",
|
|
"label": "prometheus",
|
|
"description": "",
|
|
"type": "datasource",
|
|
"pluginId": "prometheus",
|
|
"pluginName": "Prometheus"
|
|
}
|
|
],
|
|
"__elements": [],
|
|
"__requires": [
|
|
{
|
|
"type": "grafana",
|
|
"id": "grafana",
|
|
"name": "Grafana",
|
|
"version": "8.3.2"
|
|
},
|
|
{
|
|
"type": "panel",
|
|
"id": "graph",
|
|
"name": "Graph (old)",
|
|
"version": ""
|
|
},
|
|
{
|
|
"type": "datasource",
|
|
"id": "prometheus",
|
|
"name": "Prometheus",
|
|
"version": "1.0.0"
|
|
}
|
|
],
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": "-- Grafana --",
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [],
|
|
"type": "dashboard"
|
|
},
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 0,
|
|
"id": null,
|
|
"iteration": 1654687349785,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 123248,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of searching vectors.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123250,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Vector Count Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of inserting vectors.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 1
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123315,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Insert Vector Count Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for search request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 1
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123264,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_lantency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for waiting search result over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123348,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_lantency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}{{query_type}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Wait Search Result Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for reducing search result over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123316,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Reduce Search Result Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for decoding search result over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123317,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Decode Search Result Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123319,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Msg Stream Object Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for mutation request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 13
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123320,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Mutation Req Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for sending mutation over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 13
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123321,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Mutation Send Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Average cache hits per minute of cache operations.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 19
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123322,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/2) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(cache_name, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{cache_name}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Cache Hit rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"max": "1",
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for updating cache over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 19
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123323,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Cache Update Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for all physical channel.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 19
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123325,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sync Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 12,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for applying PK over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123326,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Apply PK Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for applying over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 25
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123327,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Apply Timestamp Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of DQL requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 25
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123328,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_dql_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(function_name, status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{function_name}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DQL Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of DML requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 31
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123329,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_dml_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(function_name, status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{function_name}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DML Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of DDL requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 31
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123330,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(function_name, status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{function_name}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DDL Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for DQL request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 31
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123331,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, function_name, pod, node_id) (rate(milvus_proxy_dql_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{function_name}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DQL Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for DML request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 37
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123332,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, function_name, pod, node_id) (rate(milvus_proxy_dml_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{function_name}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DML Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for DDL request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 37
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123333,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, function_name, pod, node_id) (rate(milvus_proxy_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{function_name}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DDL Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Proxy",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 123155,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of proxy nodes which has register with etcd",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 2
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123141,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Proxy Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": " Timestamps for each physical channel",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 2
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123335,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "avg time tick",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "max time tick",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "min time tick",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sync Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 12,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of DDL request",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 2
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123347,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, function_name)",
|
|
"interval": "",
|
|
"legendFormat": "{{function_name}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DDL Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1456",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1457",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for DDL request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123337,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{function_name}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DDL Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for rootcoord to finish synchronizing timestamp messages to all pchanels.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123338,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sync Timetick Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of IDs assigned by RootCoord",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123221,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2)",
|
|
"interval": "",
|
|
"legendFormat": "total",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "ID Alloc Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1456",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1457",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "RoootCoord current latest timestamp",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123339,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "tiestamp",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Timestamp",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "RootCoord stores pre-assigned timestamps in the metastore",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123340,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "tiestamp",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Timestamp Saved",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of collections.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123342,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of partitions.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 20
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123344,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Partition Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of DML channels",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 20
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123345,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DML Channel Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of Msgstream objects.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 20
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123346,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Msgstream Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of credential",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123343,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Credential Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Root Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 2
|
|
},
|
|
"id": 123246,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of loaded collections.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 3
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123288,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of loaded entities.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 3
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123289,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Entity Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of load requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 3
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123291,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status)",
|
|
"interval": "",
|
|
"legendFormat": "{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of relaase requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123292,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querycoord_release_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status)",
|
|
"interval": "",
|
|
"legendFormat": "{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Release Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for load request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123294,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for DQL release over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123313,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Release Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of sub-load task.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123295,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_child_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sub-Load Task",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number pf parent loading task.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123296,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_parent_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Parent Load Task",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for sub-load task request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123298,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sub-Load Task Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of Query nodes which has register with etcd.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 21
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123297,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Query Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Query Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 3
|
|
},
|
|
"id": 123244,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of the loaded collections in QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123299,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of the loaded partitions in QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 4
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123303,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Partition Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of the loaded segment in QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 4
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123305,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Segment Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of dml virtual channels for QueryNode watch",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123304,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DML Virtual Channel",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of delta virtual channels for QueryNode watch",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 10
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123306,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Delta Virtual Channel",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of consumers in the QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 10
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123307,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Consumer Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of searching requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123350,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (query_type, status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for searching segment over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 16
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123300,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": false,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, segment_state, pod, node_id) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"instant": true,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{segment_state}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Segment Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for searching at the segcore step over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 16
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123310,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Segcore Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for loading segment over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123311,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Segment Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of flowgraph",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 22
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123312,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flowgraph Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The length of the task queue for unsolved read requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 22
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123351,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Unsolved Read Task Length",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The length of the task queue of read requests to be executed",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 28
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123356,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Ready Read Task Length",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of read requests currently being executed in parallel",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 28
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123357,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Parallel Read Task Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "cpu utilization under scheduler evaluation",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 28
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123358,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Estimate CPU Usage",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of original tasks contained in the merged search tas",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 34
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123352,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Group Size",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of queries for search requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 34
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123361,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search NQ",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of queries for the merged search requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 34
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123360,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Group NQ",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Top_K for search requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 40
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123359,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Top_K",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Top_K for the merged search requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 40
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123362,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Group Top_K",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of evicted read requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 40
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123364,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Evicted Read Requests Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Query Node",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"id": 123172,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of data nodes which has register with etcd.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123207,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "total",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Data Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of segments with different states in the Meta of DataCoord",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 5
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123267,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{segment_state}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Segment Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of collections recorded in DataCoord Meta",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 5
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123268,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "total",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of rows of valid data accumulated in DataCoord that flushed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 11
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123269,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "row",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Stored Rows",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "On the DataCoord, the average, max and min timestmp on each physical topic",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 11
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123270,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "avg time tick",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "max time tick",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "min time tick",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sync Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 12,
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Data Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"id": 123242,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of flowgraph",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123272,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flowgraph Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of messages consumed for insert and delete operation.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 6
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123274,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (msg_type, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Msg Rows Consum Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of each message that has been flushed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 6
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123275,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (msg_type, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flush Data Size Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of consumers created on the DataNode",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123276,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Consumer Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of prodecer created on the DataNode",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 12
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123277,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Producer Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "On the DataNode, the average, max and min timestmp on each physical topic",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 12
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123279,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "max-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "min-{{pod}}-{{node_id}}",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sync Time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of segment that has been not flushed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 18
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123280,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Unflushed Segment Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for encoding the data in the buffer over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 18
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123282,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Encode Buffer Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for writting the data in buffer to storage over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 18
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123283,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Save Data Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of flush operete.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123284,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flush Operate Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of auto flush operate.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 24
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123285,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Autoflush Operate Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of flush requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 24
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123286,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flush Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for compaction over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 30
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123314,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Compaction Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Data Node",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"id": 123223,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of build requests recived by IndexCoord.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123225,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexcoord_indexreq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status)",
|
|
"interval": "",
|
|
"legendFormat": "{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:372",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:373",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of indexing tasks with different states in IndexCoord's meta",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123227,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_indexcoord_indextask_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (index_task_status)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{index_task_status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Task Count",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:401",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:402",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of index nodes which has register with etcd",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123229,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_indexcoord_indexnode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "total",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:479",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:480",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Index Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"id": 123231,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-minute increasing rate of index tasks recevied by IndexNode",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123233,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Task Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for loading the FieldData over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123235,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Field Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for decoding the FieldData over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123238,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Decode Field Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:190",
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:191",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for building index over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123237,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Build Index Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:246",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:247",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for encoding index over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123239,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Encode Index Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:275",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:276",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for save index over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123240,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Save Index Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:331",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:332",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Index Node",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"id": 123157,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total cpu usage of all milvus components.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123202,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m]))",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "cpu usage",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "CPU Usage Total",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:8411",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:8412",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123204,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "memory",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory Total",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of goroutines that currently exist.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123206,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "goroutines",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Goroutines Total",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "process cpu usage",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123135,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "CPU Usage",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Process memory of milvus pod.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123133,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": false,
|
|
"expr": "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of goroutines that currently exist.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123125,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Goroutines",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of bytes allocated and still in use.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 21
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123137,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Allocated Memory",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "GC Max duration seconds.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 21
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123131,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "GC Max duration seconds",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of OS threads created.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 21
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123127,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "OS Threads",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Next GC Bytes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123129,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Next GC Bytes",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of process opened fds.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 27
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123211,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Process Opened Fds",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The fraction of this program's available CPU time used by the GC since the program started.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 27
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123212,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "GC CPU time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Rate of memory frees.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123213,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory Free Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Rate of go memory mallocs.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 33
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123215,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Memory mallocs rate.",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of allocated heap objects.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 33
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123216,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Allocated Heap Objects",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Rate of bytes allocated, even if freed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 39
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123217,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory Allocated Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:8539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:8540",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of heap bytes released to OS.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 39
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123218,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Heap released",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of heap bytes waiting to be used.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 39
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123219,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.2",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Heap idle",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Runtime",
|
|
"type": "row"
|
|
}
|
|
],
|
|
"refresh": false,
|
|
"schemaVersion": 33,
|
|
"style": "dark",
|
|
"tags": [
|
|
"milvus2.0"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "prometheus",
|
|
"value": "prometheus"
|
|
},
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "datasource",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"queryValue": "",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": "milvus",
|
|
"value": "milvus"
|
|
},
|
|
"hide": 2,
|
|
"name": "app_name",
|
|
"options": [
|
|
{
|
|
"selected": true,
|
|
"text": "milvus",
|
|
"value": "milvus"
|
|
}
|
|
],
|
|
"query": "milvus",
|
|
"skipUrlSync": false,
|
|
"type": "textbox"
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"definition": "label_values(kube_pod_info, namespace)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "namespace",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(kube_pod_info, namespace)",
|
|
"refId": "prometheus-namespace-Variable-Query"
|
|
},
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"definition": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "instance",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)",
|
|
"refId": "prometheus-instance-Variable-Query"
|
|
},
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": true,
|
|
"name": "pod",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)",
|
|
"refId": "prometheus-pod-Variable-Query"
|
|
},
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": true,
|
|
"name": "component",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)",
|
|
"refId": "StandardVariableQuery"
|
|
},
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-5m",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"hidden": false,
|
|
"refresh_intervals": [
|
|
"5s",
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
],
|
|
"time_options": [
|
|
"5m",
|
|
"15m",
|
|
"1h",
|
|
"6h",
|
|
"12h",
|
|
"24h",
|
|
"2d",
|
|
"7d",
|
|
"30d"
|
|
],
|
|
"type": "timepicker"
|
|
},
|
|
"timezone": "browser",
|
|
"title": "Milvus2.0",
|
|
"uid": "uLf5cJ3Gz",
|
|
"version": 5,
|
|
"weekStart": ""
|
|
} |