mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-02 20:09:57 +08:00
4c2b20378d
Signed-off-by: yun.zhang <yun.zhang@zilliz.com> Signed-off-by: yun.zhang <yun.zhang@zilliz.com>
12283 lines
322 KiB
JSON
12283 lines
322 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": "-- Grafana --",
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [],
|
|
"type": "dashboard"
|
|
},
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 0,
|
|
"id": 34,
|
|
"iteration": 1667987821492,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 123248,
|
|
"panels": [],
|
|
"title": "Proxy",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of searching vectors.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123250,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Vector Count Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of inserting vectors.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 1
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123315,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Insert Vector Count Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of search request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 1
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123264,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123377,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_collection_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "p99-{{query_type}}-{{pod}}-{{node_id}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_collection_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_collection_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{query_type}}-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Search Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of mutation request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123320,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Mutation Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123378,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_collection_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_collection_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Mutation Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of wait search result over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123348,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_sq_wait_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_wait_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Wait Search Result Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of reduce search result over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 13
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123316,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Reduce Search Result Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of decode search result over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 13
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123317,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Decode Search Result Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 19
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123319,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Msg Stream Object Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of send mutation over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 19
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123321,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{msg_type}}-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{msg_type}}-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Mutation Send Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Average cache hits per minute of cache operations.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 19
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123322,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(cache_name, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{cache_name}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Cache Hit rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"max": "1",
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of update cache over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123323,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Cache Update Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for time tick behind.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 25
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123325,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Produced Timetick Lag Behind Now",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for applying PK over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 25
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123326,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Apply PK Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 31
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123327,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Apply Timestamp Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of success requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 31
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123329,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[2m])/120) by(function_name, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{function_name}}-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Success Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of faild requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 31
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123379,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by(function_name, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{function_name}}-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Faild Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 37
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123381,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{function_name}}-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{function_name}}-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3538",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of bytes received in proxy",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 37
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123368,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_receive_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Received Byte Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 37
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123369,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_proxy_send_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Send Byte Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 43
|
|
},
|
|
"id": 123155,
|
|
"panels": [],
|
|
"title": "Root Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of proxy nodes which has register with etcd",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 44
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123141,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Proxy Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for time tick behind.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 44
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123384,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Produced Timetick Lag Behind Now",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 44
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123338,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-latency",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Produced Timetick Time Taken",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of DDL request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 50
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123337,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{function_name}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{function_name}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DDL Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "RootCoord stores pre-assigned timestamps in the metastore",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 50
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123340,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "tiestamp",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Timestamp Saved",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of DDL request",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 50
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123347,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)",
|
|
"interval": "",
|
|
"legendFormat": "{{function_name}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DDL Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1456",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1457",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "RoootCoord current latest timestamp",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 56
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123339,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "tiestamp",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Timestamp",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of DML channels",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 56
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123345,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DML Channel Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of IDs assigned by RootCoord",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 56
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123221,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)",
|
|
"interval": "",
|
|
"legendFormat": "total",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "ID Alloc Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1456",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1457",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of partitions.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 62
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123344,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Partition Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The max time tick delay of flow graphs. unit ms",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 62
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123383,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)", "interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{role_name}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Time Tick Delay",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of collections.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 62
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123342,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of credential",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 68
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123343,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Credential Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of Msgstream objects.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 68
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123346,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Msgstream Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:4353",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:4354",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 74
|
|
},
|
|
"id": 123246,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of loaded collections.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 3
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123288,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of loaded entities.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 3
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123289,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Entity Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of load requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 3
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123291,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)",
|
|
"interval": "",
|
|
"legendFormat": "{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of relaase requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123292,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querycoord_release_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)",
|
|
"interval": "",
|
|
"legendFormat": "{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Release Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of load request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123294,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querycoord_load_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_load_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-latency",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of release request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 9
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123313,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querycoord_release_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_release_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-latency",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Release Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of sub-load task.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123295,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_child_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sub-Load Task",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number pf parent loading task.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123296,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_parent_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Parent Load Task",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of sub-load task request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 15
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123298,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-latency",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querycoord_child_task_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_child_task_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-latency",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Sub-Load Task Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of Query nodes which has register with etcd.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 21
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123297,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "num",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Query Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Query Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 75
|
|
},
|
|
"id": 123244,
|
|
"panels": [],
|
|
"title": "Query Node",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of the loaded collections in QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 76
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123299,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Collection Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of the loaded partitions in QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 76
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123303,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Partition Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of the loaded segment in QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 76
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123305,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Segment Loaded Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for time tick behind.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 82
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123385,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "TimeTick Lag Behind Now (Consumed Insert)",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for time tick behind.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 82
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123386,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "TimeTick Lag Behind Now (Consumed Delete)",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of consuming message",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 82
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123387,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-all",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Consumed Message Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "cps",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of the queryable entities in QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 88
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123365,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Queryable Entity Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of dml virtual channels for QueryNode watch",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 88
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123304,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "DML Virtual Channel",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of delta virtual channels for QueryNode watch",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 88
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123306,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Delta Virtual Channel",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of consumers in the QueryNode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123307,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Consumer Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of searching requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123350,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{query_type}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of search and query request over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 94
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123366,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 100
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123372,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search in Queue Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of search segment over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 100
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123374,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": false,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Segment Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 100
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123310,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Segcore Request Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 106
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123367,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}-{{query_type}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Reduce Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of load segment over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 106
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123311,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Segment Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of flowgraph",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 106
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123312,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flowgraph Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The length of the task queue for unsolved read requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 112
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123351,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Unsolved Read Task Length",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The length of the task queue of read requests to be executed",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 112
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123356,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Ready Read Task Length",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of read requests currently being executed in parallel",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 112
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123357,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Parallel Read Task Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "cpu utilization under scheduler evaluation",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 118
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123358,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Estimate CPU Usage",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of original tasks contained in the merged search task",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 118
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123352,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Group Size",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of queries for search requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 118
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123361,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search NQ",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of queries for the merged search requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 124
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123360,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-milvus_querynode_search_group_nq",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Group NQ",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Top_K for search requests",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 124
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123359,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Top_K",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Top_K for the merged search requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 124
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123362,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Search Group Top_K",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of evicted read requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 130
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123364,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Evicted Read Requests Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 136
|
|
},
|
|
"id": 123172,
|
|
"panels": [],
|
|
"title": "Data Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of data nodes which has register with etcd.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 137
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123207,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "total",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Data Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 137
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123390,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "TimeTick Lag Behind Now From DataNode",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The number of rows of valid data accumulated in DataCoord that flushed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 137
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123269,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "row",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Stored Rows",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of data flushed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 143
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123371,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Stored Rows Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of segments with different states in the Meta of DataCoord",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 143
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123267,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{segment_state}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Segment Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "binlog size of all collections/segments, unit byte",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 143
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123382,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "size",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Stored Binlog Size",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 149
|
|
},
|
|
"id": 123242,
|
|
"panels": [],
|
|
"title": "Data Node",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of flowgraph",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 150
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123272,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flowgraph Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of prodecer created on the DataNode",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 150
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123277,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Producer Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 150
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123393,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "TimeTick Lag Behind Now (Consumed All)",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P1809F7CD0C75ACF3"
|
|
},
|
|
"description": "Average, maximum and minimum values of the timestamps for time tick lag behind now, which send to DataCoord",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 156
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123388,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "avg(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}-avg",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "max(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-max",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "min(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-min",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Produced TimeTick Lag Behind Now For DC",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of consuming message",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 156
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123389,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Consumed Message Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:3414",
|
|
"format": "cps",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:3415",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "forward delete and timetick message to delta channel latency",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 156
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123394,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_forward_delete_msg_time_taken_ms_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Forward Delete&Timetick Message latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"decimals": 0,
|
|
"format": "ms",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of consumers created on the DataNode",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 162
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123276,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Consumer Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 162
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123282,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Encode Buffer Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of messages consumed for insert and delete operation.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 162
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123274,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Msg Rows Consumed Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "cps",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total number of segment that has been not flushed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 168
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123280,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Unflushed Segment Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:536",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:537",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of auto flush operate.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 168
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123285,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Autoflush Operate Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of each message that has been flushed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 168
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123275,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{msg_type}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flush Data Size Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of flush operete.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 174
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123284,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flush Operate Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of flush requests.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 174
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123286,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Flush Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 174
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123283,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Save Data Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of compaction over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 180
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123314,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Compaction Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 186
|
|
},
|
|
"id": 123223,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of building index requests recived by IndexCoord.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123225,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexcoord_index_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)",
|
|
"interval": "",
|
|
"legendFormat": "{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Request Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:372",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:373",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of indexing tasks with different states in IndexCoord's meta",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123227,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_indexcoord_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (index_task_status)",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{index_task_status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Task Count",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:401",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:402",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of index nodes which has register with etcd",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 7
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123229,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(milvus_indexcoord_index_node_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "total",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Node Num",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:479",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:480",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Index Coordinator",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 187
|
|
},
|
|
"id": 123231,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "per-second increasing rate of index tasks recevied by IndexNode",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123233,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}-{{node_id}}-{{status}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Index Task Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:101",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:102",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123235,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexnode_load_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_load_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Load Field Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:161",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:162",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of decod the FieldData over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 8
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123238,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexnode_decode_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_decode_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Decode Field Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:190",
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:191",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of build index over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123237,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexnode_build_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_build_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Build Index Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:246",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:247",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile and average latency of encode index over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123239,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexnode_encode_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_encode_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Encode Index Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:275",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:276",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The 99th percentile of latency for save index over the last 2 minutes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 14
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123240,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))",
|
|
"interval": "",
|
|
"legendFormat": "p99-{{pod}}-{{node_id}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(increase(milvus_indexnode_save_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_save_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "avg-{{pod}}-{{node_id}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Save Index Latency",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:331",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:332",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Index Node",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 188
|
|
},
|
|
"id": 123157,
|
|
"panels": [
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Total cpu usage of all milvus components.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 183
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123202,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m]))",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "cpu usage",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "CPU Usage Total",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:8411",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:8412",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 183
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123204,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "memory",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory Total",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1340",
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1341",
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"show": false
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of goroutines that currently exist.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 183
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123206,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "goroutines",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Goroutines Total",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:1281",
|
|
"format": "short",
|
|
"label": "",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:1282",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "process cpu usage",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 189
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123135,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "CPU Usage",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Process memory of milvus pod.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 189
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123133,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": false,
|
|
"expr": "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"hide": false,
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of goroutines that currently exist.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 189
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123125,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Goroutines",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of bytes allocated and still in use.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 195
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123137,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Allocated Memory",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "GC Max duration seconds.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 195
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123131,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "GC Max duration seconds",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of OS threads created.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 195
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123127,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "OS Threads",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Next GC Bytes.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 201
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123129,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Next GC Bytes",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of process opened fds.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 201
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123211,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Process Opened Fds",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "The fraction of this program's available CPU time used by the GC since the program started.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 201
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123212,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "GC CPU time",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Rate of memory frees.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 207
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123213,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory Free Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Rate of go memory mallocs.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 207
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123215,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Memory mallocs rate.",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of allocated heap objects.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 207
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123216,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Allocated Heap Objects",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Rate of bytes allocated, even if freed.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 213
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123217,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Memory Allocated Rate",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"$$hashKey": "object:8539",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"$$hashKey": "object:8540",
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of heap bytes released to OS.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 213
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123218,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Heap released",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of heap bytes waiting to be used.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 213
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123219,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Heap idle",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of heap bytes that be used.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 219
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123375,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "go_memstats_heap_inuse_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{pod}}",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Go Heap in Use",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "bytes",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
},
|
|
{
|
|
"aliasColors": {},
|
|
"bars": false,
|
|
"dashLength": 10,
|
|
"dashes": false,
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"description": "Number of container threads created.",
|
|
"fill": 1,
|
|
"fillGradient": 0,
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 219
|
|
},
|
|
"hiddenSeries": false,
|
|
"id": 123376,
|
|
"legend": {
|
|
"avg": false,
|
|
"current": false,
|
|
"max": false,
|
|
"min": false,
|
|
"show": true,
|
|
"total": false,
|
|
"values": false
|
|
},
|
|
"lines": true,
|
|
"linewidth": 1,
|
|
"nullPointMode": "null",
|
|
"options": {
|
|
"alertThreshold": true
|
|
},
|
|
"percentage": false,
|
|
"pluginVersion": "8.3.3",
|
|
"pointradius": 2,
|
|
"points": false,
|
|
"renderer": "flot",
|
|
"seriesOverrides": [],
|
|
"spaceLength": 10,
|
|
"stack": false,
|
|
"steppedLine": false,
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$datasource"
|
|
},
|
|
"exemplar": true,
|
|
"expr": "container_threads{image=\"\", namespace=\"$namespace\"}",
|
|
"interval": "",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "",
|
|
"queryType": "randomWalk",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"thresholds": [],
|
|
"timeRegions": [],
|
|
"title": "Container Threads",
|
|
"tooltip": {
|
|
"shared": true,
|
|
"sort": 0,
|
|
"value_type": "individual"
|
|
},
|
|
"type": "graph",
|
|
"xaxis": {
|
|
"mode": "time",
|
|
"show": true,
|
|
"values": []
|
|
},
|
|
"yaxes": [
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"min": "0",
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"logBase": 1,
|
|
"show": true
|
|
}
|
|
],
|
|
"yaxis": {
|
|
"align": false
|
|
}
|
|
}
|
|
],
|
|
"title": "Runtime",
|
|
"type": "row"
|
|
}
|
|
],
|
|
"refresh": false,
|
|
"schemaVersion": 34,
|
|
"style": "dark",
|
|
"tags": [
|
|
"milvus2.0"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "prometheus",
|
|
"value": "prometheus"
|
|
},
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "datasource",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"queryValue": "",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": "milvus",
|
|
"value": "milvus"
|
|
},
|
|
"hide": 2,
|
|
"name": "app_name",
|
|
"options": [
|
|
{
|
|
"selected": true,
|
|
"text": "milvus",
|
|
"value": "milvus"
|
|
}
|
|
],
|
|
"query": "milvus",
|
|
"skipUrlSync": false,
|
|
"type": "textbox"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": "chaos-testing",
|
|
"value": "chaos-testing"
|
|
},
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"definition": "label_values(kube_pod_info, namespace)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "namespace",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(kube_pod_info, namespace)",
|
|
"refId": "prometheus-namespace-Variable-Query"
|
|
},
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "bulk-insert-test",
|
|
"value": "bulk-insert-test"
|
|
},
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"definition": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"multi": false,
|
|
"name": "instance",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)",
|
|
"refId": "StandardVariableQuery"
|
|
},
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"allValue": ".*",
|
|
"current": {
|
|
"selected": false,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "prometheus"
|
|
},
|
|
"definition": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)",
|
|
"hide": 0,
|
|
"includeAll": true,
|
|
"multi": false,
|
|
"name": "collection",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)",
|
|
"refId": "StandardVariableQuery"
|
|
},
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 0,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps",
|
|
"value": "bulk-insert-test-milvus-standalone-55968cfc55-cxnps"
|
|
},
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": true,
|
|
"name": "pod",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)",
|
|
"refId": "prometheus-pod-Variable-Query"
|
|
},
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"current": {
|
|
"isNone": true,
|
|
"selected": false,
|
|
"text": "None",
|
|
"value": ""
|
|
},
|
|
"datasource": {
|
|
"uid": "$datasource"
|
|
},
|
|
"definition": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)",
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"multi": true,
|
|
"name": "component",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)",
|
|
"refId": "StandardVariableQuery"
|
|
},
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"tagValuesQuery": "",
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-3h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"hidden": false,
|
|
"refresh_intervals": [
|
|
"5s",
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
],
|
|
"time_options": [
|
|
"5m",
|
|
"15m",
|
|
"1h",
|
|
"6h",
|
|
"12h",
|
|
"24h",
|
|
"2d",
|
|
"7d",
|
|
"30d"
|
|
],
|
|
"type": "timepicker"
|
|
},
|
|
"timezone": "browser",
|
|
"title": "Milvus2.0",
|
|
"uid": "uLf5cJ3Gz",
|
|
"version": 5,
|
|
"weekStart": ""
|
|
} |