2021-07-23 15:44:12 +08:00
{
2021-09-09 17:20:00 +08:00
"annotations" : {
"list" : [
2021-07-23 15:44:12 +08:00
{
2021-09-09 17:20:00 +08:00
"builtIn" : 1 ,
"datasource" : "-- Grafana --" ,
"enable" : true ,
"hide" : true ,
"iconColor" : "rgba(0, 211, 255, 1)" ,
"name" : "Annotations & Alerts" ,
2022-05-09 21:47:53 +08:00
"target" : {
"limit" : 100 ,
"matchAny" : false ,
"tags" : [ ] ,
"type" : "dashboard"
} ,
2021-09-09 17:20:00 +08:00
"type" : "dashboard"
}
]
} ,
"editable" : true ,
2022-05-09 21:47:53 +08:00
"fiscalYearStartMonth" : 0 ,
2021-09-09 17:20:00 +08:00
"graphTooltip" : 0 ,
2022-11-07 10:15:02 +08:00
"id" : 34 ,
2022-11-11 14:23:06 +08:00
"iteration" : 1667987821492 ,
2021-09-09 17:20:00 +08:00
"links" : [ ] ,
2022-05-09 21:47:53 +08:00
"liveNow" : false ,
2021-09-09 17:20:00 +08:00
"panels" : [
{
2022-10-25 15:31:30 +08:00
"collapsed" : false ,
2021-09-09 17:20:00 +08:00
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2021-12-03 09:21:32 +08:00
"y" : 0
2021-09-09 17:20:00 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123248 ,
2022-10-25 15:31:30 +08:00
"panels" : [ ] ,
"title" : "Proxy" ,
"type" : "row"
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of searching vectors." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 1
} ,
"hiddenSeries" : false ,
"id" : 123250 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
2022-05-09 21:47:53 +08:00
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Vector Count Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-09-06 18:59:11 +08:00
} ,
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of inserting vectors." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 1
} ,
"hiddenSeries" : false ,
"id" : 123315 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Insert Vector Count Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-19 15:09:28 +08:00
} ,
2022-09-06 18:59:11 +08:00
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of search request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 1
} ,
"hiddenSeries" : false ,
"id" : 123264 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "p99-{{query_type}}-{{pod}}-{{node_id}}" ,
"refId" : "A"
2022-09-06 18:59:11 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{query_type}}-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-19 15:09:28 +08:00
} ,
2022-09-06 18:59:11 +08:00
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123377 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_collection_sq_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "p99-{{query_type}}-{{pod}}-{{node_id}}" ,
"refId" : "A"
2022-09-06 18:59:11 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_collection_sq_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_collection_sq_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{query_type}}-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Search Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-19 15:09:28 +08:00
} ,
2022-09-06 18:59:11 +08:00
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of mutation request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123320 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{msg_type}}-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-09-13 19:36:57 +08:00
} ,
2021-09-09 17:34:01 +08:00
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{msg_type}}-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Mutation Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-09-06 18:59:11 +08:00
} ,
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123378 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_collection_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{msg_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-09-06 18:59:11 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_collection_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", collection_name=~\"$collection\"}[2m])) by (pod, node_id, msg_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{msg_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Mutation Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-09-09 17:20:00 +08:00
} ,
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of wait search result over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 13
} ,
"hiddenSeries" : false ,
"id" : 123348 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "A"
2022-09-06 18:59:11 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_sq_wait_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_wait_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Wait Search Result Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-09-06 18:59:11 +08:00
} ,
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of reduce search result over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 13
} ,
"hiddenSeries" : false ,
"id" : 123316 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Reduce Search Result Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-19 15:09:28 +08:00
} ,
2022-09-06 18:59:11 +08:00
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of decode search result over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 13
} ,
"hiddenSeries" : false ,
"id" : 123317 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-09-15 17:15:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_sq_decode_result_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Decode Search Result Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-19 15:09:28 +08:00
} ,
2022-09-06 18:59:11 +08:00
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Average, maximum and minimum values of the number of msgstream objects created on all physical topics." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 19
} ,
"hiddenSeries" : false ,
"id" : 123319 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-06 14:46:21 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-19 15:09:28 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Msg Stream Object Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-19 15:09:28 +08:00
} ,
2022-09-06 18:59:11 +08:00
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of send mutation over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 19
} ,
"hiddenSeries" : false ,
"id" : 123321 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-09-06 18:59:11 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{msg_type}}-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-06 14:46:21 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_mutation_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, msg_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{msg_type}}-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Mutation Send Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-19 15:09:28 +08:00
} ,
2022-09-06 18:59:11 +08:00
{
2022-10-25 15:31:30 +08:00
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Average cache hits per minute of cache operations." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 19
} ,
"hiddenSeries" : false ,
"id" : 123322 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-09-06 18:59:11 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-07-06 14:46:21 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/120) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(cache_name, pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-{{cache_name}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Cache Hit rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"max" : "1" ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of update cache over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 25
} ,
"hiddenSeries" : false ,
"id" : 123323 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-19 15:09:28 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-19 15:09:28 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_cache_update_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_cache_update_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Cache Update Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
2022-11-07 10:15:02 +08:00
"uid" : "P1809F7CD0C75ACF3"
2022-10-25 15:31:30 +08:00
} ,
2022-11-07 10:15:02 +08:00
"description" : "Average, maximum and minimum values of the timestamps for time tick behind." ,
2022-10-25 15:31:30 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 25
} ,
"hiddenSeries" : false ,
"id" : 123325 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "avg(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
2022-10-25 15:31:30 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "max(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-10-25 15:31:30 +08:00
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "min(milvus_proxy_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-10-25 15:31:30 +08:00
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
2022-11-07 10:15:02 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "P1809F7CD0C75ACF3"
} ,
"exemplar" : true ,
"expr" : "" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "" ,
"refId" : "D"
2022-10-25 15:31:30 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Produced Timetick Lag Behind Now" ,
2022-10-25 15:31:30 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
2022-11-07 10:15:02 +08:00
"decimals" : 0 ,
"format" : "ms" ,
2022-10-25 15:31:30 +08:00
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for applying PK over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 25
} ,
"hiddenSeries" : false ,
"id" : 123326 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_apply_pk_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_pk_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Apply PK Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of apply timestamp over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 31
} ,
"hiddenSeries" : false ,
"id" : 123327 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-19 15:09:28 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_apply_timestamp_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id) / sum(increase(milvus_proxy_apply_timestamp_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Apply Timestamp Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of success requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 31
} ,
"hiddenSeries" : false ,
"id" : 123329 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"success\"}[2m])/120) by(function_name, pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{function_name}}-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Success Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of faild requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 31
} ,
"hiddenSeries" : false ,
"id" : 123379 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", status=\"fail\"}[2m])/120) by(function_name, pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{function_name}}-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Faild Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 37
} ,
"hiddenSeries" : false ,
"id" : 123381 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-19 15:09:28 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id, function_name) (rate(milvus_proxy_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{function_name}}-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-09-06 18:59:11 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-07-06 14:46:21 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name) / sum(increase(milvus_proxy_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (pod, node_id, function_name)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{function_name}}-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of bytes received in proxy" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 37
} ,
"hiddenSeries" : false ,
"id" : 123368 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-19 15:09:28 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_receive_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Received Byte Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 37
} ,
"hiddenSeries" : false ,
"id" : 123369 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-19 15:09:28 +08:00
} ,
2022-10-25 15:31:30 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_proxy_send_bytes_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by(pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-09-06 18:59:11 +08:00
}
] ,
2022-10-25 15:31:30 +08:00
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Send Byte Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-15 17:15:49 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"collapsed" : false ,
2021-09-15 17:15:49 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
2021-12-07 16:15:03 +08:00
"x" : 0 ,
2022-10-25 15:31:30 +08:00
"y" : 43
2021-09-15 17:15:49 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123155 ,
2022-11-07 10:15:02 +08:00
"panels" : [ ] ,
"title" : "Root Coordinator" ,
"type" : "row"
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of proxy nodes which has register with etcd" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 44
} ,
"hiddenSeries" : false ,
"id" : 123141 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
2022-05-09 21:47:53 +08:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Proxy Node Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "P1809F7CD0C75ACF3"
} ,
"description" : "Average, maximum and minimum values of the timestamps for time tick behind." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 44
} ,
"hiddenSeries" : false ,
"id" : 123384 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "avg(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "max(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "min(milvus_rootcoord_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Produced Timetick Lag Behind Now" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "ms" ,
"logBase" : 1 ,
"show" : true
2021-09-15 17:15:49 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 44
} ,
"hiddenSeries" : false ,
"id" : 123338 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-latency" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-05-09 21:47:53 +08:00
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-latency" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Produced Timetick Time Taken" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "ms" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of DDL request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 50
} ,
"hiddenSeries" : false ,
"id" : 123337 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{function_name}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_rootcoord_ddl_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by (function_name)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{function_name}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DDL Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-09-15 17:15:49 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "RootCoord stores pre-assigned timestamps in the metastore" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 50
} ,
"hiddenSeries" : false ,
"id" : 123340 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "tiestamp" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Timestamp Saved" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of DDL request" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 50
} ,
"hiddenSeries" : false ,
"id" : 123347 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, function_name)" ,
"interval" : "" ,
"legendFormat" : "{{function_name}}-{{status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DDL Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:1456" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:1457" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "RoootCoord current latest timestamp" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 56
} ,
"hiddenSeries" : false ,
"id" : 123339 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "tiestamp" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Timestamp" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of DML channels" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 56
} ,
"hiddenSeries" : false ,
"id" : 123345 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DML Channel Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of IDs assigned by RootCoord" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 56
} ,
"hiddenSeries" : false ,
"id" : 123221 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120)" ,
"interval" : "" ,
"legendFormat" : "total" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "ID Alloc Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:1456" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:1457" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of partitions." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 62
} ,
"hiddenSeries" : false ,
"id" : 123344 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Partition Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The max time tick delay of flow graphs. unit ms" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 62
} ,
"hiddenSeries" : false ,
"id" : 123383 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_time_tick_delay{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (role_name, node_id)" , "interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{role_name}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Time Tick Delay" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of collections." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 62
} ,
"hiddenSeries" : false ,
"id" : 123342 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of credential" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 68
} ,
"hiddenSeries" : false ,
"id" : 123343 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Credential Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-15 17:15:49 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of Msgstream objects." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 68
} ,
"hiddenSeries" : false ,
"id" : 123346 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Msgstream Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-12-07 16:43:03 +08:00
} ,
2022-11-07 10:15:02 +08:00
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"collapsed" : true ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 74
} ,
"id" : 123246 ,
"panels" : [
2021-12-07 16:43:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "Total number of loaded collections." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 3
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123288 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "num" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Collection Loaded Num" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:536" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 16:43:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "Total number of loaded entities." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 3
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123289 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(milvus_querycoord_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "num" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Entity Loaded Num" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:536" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "per-second increasing rate of load requests." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-07 10:15:02 +08:00
"y" : 3
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123291 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Load Request Rate" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:101" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:102" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "per-second increasing rate of relaase requests." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 9
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123292 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(increase(milvus_querycoord_release_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Release Request Rate" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:101" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:102" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "The 99th percentile and average latency of load request over the last 2 minutes." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 9
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123294 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "p99-latency" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(increase(milvus_querycoord_load_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_load_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-latency" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Load Request Latency" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:161" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "The 99th percentile and average latency of release request over the last 2 minutes." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-07 10:15:02 +08:00
"x" : 16 ,
"y" : 9
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123313 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "p99-latency" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-11-07 10:15:02 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_querycoord_release_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_release_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-latency" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Release Request Latency" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:161" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2022-10-25 15:31:30 +08:00
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "Total number of sub-load task." ,
2022-10-25 15:31:30 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-07 10:15:02 +08:00
"x" : 0 ,
"y" : 15
2022-10-25 15:31:30 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123295 ,
2022-10-25 15:31:30 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(milvus_querycoord_child_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
2022-10-25 15:31:30 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "num" ,
2022-10-25 15:31:30 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Sub-Load Task" ,
2022-10-25 15:31:30 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:536" ,
2022-10-25 15:31:30 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
2022-10-25 15:31:30 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2022-11-07 10:15:02 +08:00
} ,
2022-05-09 21:47:53 +08:00
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "Total number pf parent loading task." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-07 10:15:02 +08:00
"x" : 8 ,
"y" : 15
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123296 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(milvus_querycoord_parent_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Parent Load Task" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 14:57:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "The 99th percentile and average latency of sub-load task request over the last 2 minutes." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-07 10:15:02 +08:00
"x" : 16 ,
"y" : 15
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123298 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-11-07 10:15:02 +08:00
"legendFormat" : "p99-latency" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-11-07 10:15:02 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_querycoord_child_task_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) / sum(increase(milvus_querycoord_child_task_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m]))" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-latency" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Sub-Load Task Latency" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:161" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"description" : "Number of Query nodes which has register with etcd." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-07 10:15:02 +08:00
"x" : 0 ,
"y" : 21
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
2022-11-07 10:15:02 +08:00
"id" : 123297 ,
2022-05-09 21:47:53 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-11-07 10:15:02 +08:00
"expr" : "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-11-07 10:15:02 +08:00
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-07 10:15:02 +08:00
"title" : "Query Node Num" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:536" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2022-11-07 10:15:02 +08:00
}
] ,
"title" : "Query Coordinator" ,
"type" : "row"
} ,
{
"collapsed" : false ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 75
} ,
"id" : 123244 ,
"panels" : [ ] ,
"title" : "Query Node" ,
"type" : "row"
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of the loaded collections in QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 76
} ,
"hiddenSeries" : false ,
"id" : 123299 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-12-07 16:43:03 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of the loaded partitions in QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 76
} ,
"hiddenSeries" : false ,
"id" : 123303 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Partition Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of the loaded segment in QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 76
} ,
"hiddenSeries" : false ,
"id" : 123305 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
2023-02-10 10:16:31 +08:00
"stack" : true ,
2022-11-07 10:15:02 +08:00
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2023-02-10 10:16:31 +08:00
"expr" : "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id, segment_state)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2023-02-10 10:16:31 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{segment_state}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Segment Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-12-07 16:43:03 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "P1809F7CD0C75ACF3"
} ,
"description" : "Average, maximum and minimum values of the timestamps for time tick behind." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 82
} ,
"hiddenSeries" : false ,
"id" : 123385 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-05-09 21:47:53 +08:00
} ,
2021-12-07 14:57:03 +08:00
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"insert\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "TimeTick Lag Behind Now (Consumed Insert)" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "ms" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "P1809F7CD0C75ACF3"
} ,
"description" : "Average, maximum and minimum values of the timestamps for time tick behind." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 82
} ,
"hiddenSeries" : false ,
"id" : 123386 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "avg(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "max(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "min(milvus_querynode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"delete\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "TimeTick Lag Behind Now (Consumed Delete)" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "ms" ,
"logBase" : 1 ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-12-07 16:43:03 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of consuming message" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 82
} ,
"hiddenSeries" : false ,
"id" : 123387 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-all" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Consumed Message Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "cps" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
2021-12-07 16:43:03 +08:00
}
] ,
2022-11-07 10:15:02 +08:00
"yaxis" : {
"align" : false
}
2021-12-07 14:57:03 +08:00
} ,
2021-09-15 17:15:49 +08:00
{
2022-11-07 10:15:02 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of the queryable entities in QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
2021-09-15 17:15:49 +08:00
"gridPos" : {
2022-11-07 10:15:02 +08:00
"h" : 6 ,
"w" : 8 ,
2022-05-09 21:47:53 +08:00
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 88
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"hiddenSeries" : false ,
"id" : 123365 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
2022-05-09 21:47:53 +08:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Queryable Entity Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-09-15 17:15:49 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of dml virtual channels for QueryNode watch" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 88
} ,
"hiddenSeries" : false ,
"id" : 123304 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DML Virtual Channel" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-12-07 16:43:03 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of delta virtual channels for QueryNode watch" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 88
} ,
"hiddenSeries" : false ,
"id" : 123306 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Delta Virtual Channel" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of consumers in the QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 94
} ,
"hiddenSeries" : false ,
"id" : 123307 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Consumer Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-09-15 17:15:49 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of searching requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 94
} ,
"hiddenSeries" : false ,
"id" : 123350 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (query_type, status, pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-{{query_type}}-{{status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-07-06 14:46:21 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of search and query request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 94
} ,
"hiddenSeries" : false ,
"id" : 123366 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-07-06 14:46:21 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-06 14:46:21 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_sq_req_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-07-06 14:46:21 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of search or query in queue over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 100
} ,
"hiddenSeries" : false ,
"id" : 123372 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-07-06 14:46:21 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-06 14:46:21 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_sq_queue_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search in Queue Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-07-23 15:44:12 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of search segment over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 100
} ,
"hiddenSeries" : false ,
"id" : 123374 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : false ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
2023-02-10 10:16:31 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}-{{segment_state}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2021-07-23 15:44:12 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_sq_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type, segment_state)" ,
"hide" : false ,
"interval" : "" ,
2023-02-10 10:16:31 +08:00
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}_{{segment_state}}" ,
2022-11-07 10:15:02 +08:00
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Segment Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-07-23 15:44:12 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of search at the segcore step over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 100
} ,
"hiddenSeries" : false ,
"id" : 123310 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-05-31 09:02:03 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_sq_core_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Segcore Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-11-01 19:23:48 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of search or query reduce over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 106
} ,
"hiddenSeries" : false ,
"id" : 123367 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}-{{query_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-07-23 15:44:12 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_sq_reduce_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id, query_type)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}-{{query_type}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Reduce Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-09-09 17:20:00 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of load segment over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 106
} ,
"hiddenSeries" : false ,
"id" : 123311 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_load_segment_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_load_segment_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Load Segment Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of flowgraph" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 106
} ,
"hiddenSeries" : false ,
"id" : 123312 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Flowgraph Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-06-10 10:20:07 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The length of the task queue for unsolved read requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 112
} ,
"hiddenSeries" : false ,
"id" : 123351 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-06-10 10:20:07 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Unsolved Read Task Length" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The length of the task queue of read requests to be executed" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 112
} ,
"hiddenSeries" : false ,
"id" : 123356 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-06-10 10:20:07 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Ready Read Task Length" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-06-10 10:20:07 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of read requests currently being executed in parallel" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 112
} ,
"hiddenSeries" : false ,
"id" : 123357 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-06-10 10:20:07 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Parallel Read Task Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-06-10 10:20:07 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "cpu utilization under scheduler evaluation" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 118
} ,
"hiddenSeries" : false ,
"id" : 123358 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-06-10 10:20:07 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Estimate CPU Usage" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-06-10 10:20:07 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of original tasks contained in the merged search task" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 118
} ,
"hiddenSeries" : false ,
"id" : 123352 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-06-10 10:20:07 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-06-10 10:20:07 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_search_group_size_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_size_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Group Size" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-06-10 10:20:07 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of queries for search requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 118
} ,
"hiddenSeries" : false ,
"id" : 123361 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-06-10 10:20:07 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-06-10 10:20:07 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_search_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search NQ" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-06-10 10:20:07 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of queries for the merged search requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 124
} ,
"hiddenSeries" : false ,
"id" : 123360 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-06-10 10:20:07 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-06-10 10:20:07 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_search_group_nq_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_nq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-milvus_querynode_search_group_nq" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Group NQ" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Top_K for search requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 124
} ,
"hiddenSeries" : false ,
"id" : 123359 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-06-10 10:20:07 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-06-10 10:20:07 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Top_K" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Top_K for the merged search requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 124
} ,
"hiddenSeries" : false ,
"id" : 123362 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-06-10 10:20:07 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-06-10 10:20:07 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_search_group_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_querynode_search_group_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Group Top_K" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-06-10 10:20:07 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of evicted read requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 130
} ,
"hiddenSeries" : false ,
"id" : 123364 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-06-10 10:20:07 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-07-23 15:44:12 +08:00
}
2021-09-09 17:20:00 +08:00
] ,
2022-11-07 10:15:02 +08:00
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Evicted Read Requests Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
2023-08-18 16:20:18 +08:00
{
"id" : 123397 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 130
} ,
"type" : "graph" ,
"title" : "Knowhere Search Top_K" ,
"datasource" : {
"uid" : "$datasource" ,
"type" : "prometheus"
} ,
"thresholds" : [ ] ,
"pluginVersion" : "8.5.20" ,
"description" : "Top_K for knowhere search requests" ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"aliasColors" : { } ,
"dashLength" : 10 ,
"fill" : 1 ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"pointradius" : 2 ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(knowhere_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(knowhere_search_topk_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(knowhere_search_topk_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"timeRegions" : [ ] ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ] ,
"name" : null ,
"buckets" : null
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
} ,
"bars" : false ,
"dashes" : false ,
"fillGradient" : 0 ,
"hiddenSeries" : false ,
"percentage" : false ,
"points" : false ,
"stack" : false ,
"steppedLine" : false ,
"timeFrom" : null ,
"timeShift" : null
} ,
{
"id" : 123395 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 130
} ,
"type" : "graph" ,
"title" : "Knowhere Search Count" ,
"datasource" : {
"uid" : "$datasource" ,
"type" : "prometheus"
} ,
"thresholds" : [ ] ,
"pluginVersion" : "8.5.20" ,
"description" : "Total number of knowhere search cnt." ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"aliasColors" : { } ,
"dashLength" : 10 ,
"fill" : 1 ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"pointradius" : 2 ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(knowhere_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"timeRegions" : [ ] ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ] ,
"name" : null ,
"buckets" : null
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
} ,
"bars" : false ,
"dashes" : false ,
"fillGradient" : 0 ,
"hiddenSeries" : false ,
"percentage" : false ,
"points" : false ,
"stack" : false ,
"steppedLine" : false ,
"timeFrom" : null ,
"timeShift" : null
} ,
{
"id" : 123396 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 136
} ,
"type" : "graph" ,
"title" : "Knowhere Range Search Count" ,
"datasource" : {
"uid" : "$datasource" ,
"type" : "prometheus"
} ,
"thresholds" : [ ] ,
"pluginVersion" : "8.5.20" ,
"description" : "Total number of knowhere range search cnt." ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"aliasColors" : { } ,
"dashLength" : 10 ,
"fill" : 1 ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"pointradius" : 2 ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(knowhere_range_search_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"timeRegions" : [ ] ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ] ,
"name" : null ,
"buckets" : null
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
} ,
"bars" : false ,
"dashes" : false ,
"fillGradient" : 0 ,
"hiddenSeries" : false ,
"percentage" : false ,
"points" : false ,
"stack" : false ,
"steppedLine" : false ,
"timeFrom" : null ,
"timeShift" : null
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-07 10:15:02 +08:00
"collapsed" : false ,
2021-09-09 17:20:00 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
2021-09-09 17:20:00 +08:00
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 136
2021-09-09 17:20:00 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123172 ,
2022-11-07 10:15:02 +08:00
"panels" : [ ] ,
"title" : "Data Coordinator" ,
"type" : "row"
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of data nodes which has register with etcd." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 137
} ,
"hiddenSeries" : false ,
"id" : 123207 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "total" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Data Node Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "P1809F7CD0C75ACF3"
} ,
"description" : "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 137
} ,
"hiddenSeries" : false ,
"id" : 123390 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
2022-05-09 21:47:53 +08:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "avg(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "max(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "min(milvus_datacoord_consume_datanode_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "TimeTick Lag Behind Now From DataNode" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "ms" ,
"logBase" : 1 ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of rows of valid data accumulated in DataCoord that flushed." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 137
} ,
"hiddenSeries" : false ,
"id" : 123269 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "row" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Stored Rows" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-09-09 17:20:00 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of data flushed." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 143
} ,
"hiddenSeries" : false ,
"id" : 123371 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_datacoord_stored_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Stored Rows Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-07-06 14:46:21 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of segments with different states in the Meta of DataCoord" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 143
} ,
"hiddenSeries" : false ,
"id" : 123267 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-07-06 14:46:21 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{segment_state}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Segment Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "binlog size of all collections/segments, unit byte" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 143
} ,
"hiddenSeries" : false ,
"id" : 123382 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-10-25 15:31:30 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_stored_binlog_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "size" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Stored Binlog Size" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-10-25 15:31:30 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"collapsed" : false ,
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 149
} ,
"id" : 123242 ,
"panels" : [ ] ,
"title" : "Data Node" ,
"type" : "row"
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of flowgraph" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 150
} ,
"hiddenSeries" : false ,
"id" : 123272 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-10-25 15:31:30 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Flowgraph Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of prodecer created on the DataNode" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 150
} ,
"hiddenSeries" : false ,
"id" : 123277 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Producer Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
2021-09-09 17:20:00 +08:00
}
] ,
2022-11-07 10:15:02 +08:00
"yaxis" : {
"align" : false
}
2021-07-23 15:44:12 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-07 10:15:02 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "P1809F7CD0C75ACF3"
} ,
"description" : "Average, maximum and minimum values of the timestamps for time tick lag behind now." ,
"fill" : 1 ,
"fillGradient" : 0 ,
2021-09-09 17:20:00 +08:00
"gridPos" : {
2022-11-07 10:15:02 +08:00
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 150
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"hiddenSeries" : false ,
"id" : 123393 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
2022-05-09 21:47:53 +08:00
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "avg(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "max(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "min(milvus_datanode_consume_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", msg_type=\"all\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "TimeTick Lag Behind Now (Consumed All)" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "ms" ,
"logBase" : 1 ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "P1809F7CD0C75ACF3"
} ,
"description" : "Average, maximum and minimum values of the timestamps for time tick lag behind now, which send to DataCoord" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 156
} ,
"hiddenSeries" : false ,
"id" : 123388 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "avg(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "max(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "min(milvus_datanode_produce_tt_lag_ms{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Produced TimeTick Lag Behind Now For DC" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "ms" ,
"logBase" : 1 ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-second increasing rate of consuming message" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 156
} ,
"hiddenSeries" : false ,
"id" : 123389 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_datanode_consume_msg_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (pod, node_id, msg_type)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Consumed Message Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "cps" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "forward delete and timetick message to delta channel latency" ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 16 ,
"y" : 156
2022-11-07 10:15:02 +08:00
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123394 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_forward_delete_msg_time_taken_ms_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"hide" : false ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_forward_delete_msg_time_taken_ms_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "C"
2022-11-07 10:15:02 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Forward Delete&Timetick Message latency" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
2022-11-11 14:23:06 +08:00
"decimals" : 0 ,
"format" : "ms" ,
2022-11-07 10:15:02 +08:00
"logBase" : 1 ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "Total number of consumers created on the DataNode" ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 162
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123276 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Consumer Num" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:536" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:537" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 162
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123282 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-11-11 14:23:06 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_datanode_encode_buffer_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_encode_buffer_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
2022-11-07 10:15:02 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Encode Buffer Latency" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:161" ,
"format" : "short" ,
2022-11-07 10:15:02 +08:00
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:162" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "per-second increasing rate of messages consumed for insert and delete operation." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 16 ,
"y" : 162
2022-11-07 10:15:02 +08:00
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123274 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Msg Rows Consumed Rate" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:101" ,
"format" : "cps" ,
2022-11-07 10:15:02 +08:00
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:102" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "Total number of segment that has been not flushed." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 168
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123280 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Unflushed Segment Num" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:536" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:537" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "per-second increasing rate of auto flush operate." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 168
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123285 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Autoflush Operate Rate" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
2022-11-11 14:23:06 +08:00
"format" : "short" ,
2022-11-07 10:15:02 +08:00
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "per-second increasing rate of each message that has been flushed." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 16 ,
"y" : 168
2022-11-07 10:15:02 +08:00
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123275 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (msg_type, pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Flush Data Size Rate" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2021-09-09 17:20:00 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "per-second increasing rate of flush operete." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 174
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123284 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Flush Operate Rate" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "per-second increasing rate of flush requests." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 8 ,
"y" : 174
2022-11-07 10:15:02 +08:00
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123286 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)" ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Flush Request Rate" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:101" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
2022-05-09 21:47:53 +08:00
} ,
2021-09-14 14:53:48 +08:00
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:102" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-11 14:23:06 +08:00
"y" : 174
2022-11-07 10:15:02 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123283 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
2021-09-14 14:53:48 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
"expr" : "sum(increase(milvus_datanode_save_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_save_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Save Data Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-11-11 14:23:06 +08:00
"description" : "The 99th percentile and average latency of compaction over the last 2 minutes." ,
2022-11-07 10:15:02 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-11-11 14:23:06 +08:00
"x" : 0 ,
"y" : 180
2022-11-07 10:15:02 +08:00
} ,
"hiddenSeries" : false ,
2022-11-11 14:23:06 +08:00
"id" : 123314 ,
2022-11-07 10:15:02 +08:00
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
2022-11-07 10:15:02 +08:00
"exemplar" : true ,
2022-11-11 14:23:06 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"hide" : false ,
2022-11-07 10:15:02 +08:00
"interval" : "" ,
2022-11-11 14:23:06 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
2022-11-07 10:15:02 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-11-11 14:23:06 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_datanode_compaction_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_datanode_compaction_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
2021-09-14 14:53:48 +08:00
}
] ,
2022-11-07 10:15:02 +08:00
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-11-11 14:23:06 +08:00
"title" : "Compaction Latency" ,
2022-11-07 10:15:02 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:161" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-11 14:23:06 +08:00
"$$hashKey" : "object:162" ,
2022-11-07 10:15:02 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 15:57:47 +08:00
} ,
{
2022-10-25 15:31:30 +08:00
"collapsed" : true ,
2021-09-14 15:57:47 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2022-11-11 14:23:06 +08:00
"y" : 186
2022-05-09 21:47:53 +08:00
} ,
"id" : 123223 ,
2022-10-25 15:31:30 +08:00
"panels" : [
2022-05-09 21:47:53 +08:00
{
2022-10-25 15:31:30 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"description" : "per-second increasing rate of building index requests recived by IndexCoord." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 7
2022-05-09 21:47:53 +08:00
} ,
2022-10-25 15:31:30 +08:00
"hiddenSeries" : false ,
"id" : 123225 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_indexcoord_index_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status)" ,
"interval" : "" ,
"legendFormat" : "{{status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Index Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:372" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:373" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2022-10-19 15:09:28 +08:00
{
2022-10-25 15:31:30 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
2022-05-09 21:47:53 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-10-25 15:31:30 +08:00
"description" : "Number of indexing tasks with different states in IndexCoord's meta" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123227 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_indexcoord_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (index_task_status)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{index_task_status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Index Task Count" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:401" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:402" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 15:57:47 +08:00
} ,
{
2022-10-25 15:31:30 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of index nodes which has register with etcd" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123229 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_indexcoord_index_node_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"hide" : false ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "total" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Index Node Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:479" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:480" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 15:57:47 +08:00
}
] ,
2022-10-25 15:31:30 +08:00
"title" : "Index Coordinator" ,
"type" : "row"
2021-09-14 16:59:48 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-14 16:59:48 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
2021-09-14 16:59:48 +08:00
"x" : 0 ,
2022-11-11 14:23:06 +08:00
"y" : 187
2021-09-14 16:59:48 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123231 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-09-06 18:59:11 +08:00
"description" : "per-second increasing rate of index tasks recevied by IndexNode" ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-07-19 08:54:28 +08:00
"y" : 8
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123233 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-09-06 18:59:11 +08:00
"expr" : "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/120) by (status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-05-23 18:09:59 +08:00
"title" : "Index Task Rate" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-14 16:59:48 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-07-19 08:54:28 +08:00
"description" : "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-07-19 08:54:28 +08:00
"y" : 8
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123235 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-07-19 08:54:28 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-19 08:54:28 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_indexnode_load_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_load_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Load Field Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-07-19 08:54:28 +08:00
"description" : "The 99th percentile and average latency of decod the FieldData over the last 2 minutes." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-07-19 08:54:28 +08:00
"y" : 8
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123238 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-07-19 08:54:28 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-19 08:54:28 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_indexnode_decode_field_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_decode_field_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Decode Field Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:190" ,
"format" : "short" ,
"label" : "" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:191" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-07-19 08:54:28 +08:00
"description" : "The 99th percentile and average latency of build index over the last 2 minutes." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-07-19 08:54:28 +08:00
"y" : 14
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123237 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-07-19 08:54:28 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-19 08:54:28 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_indexnode_build_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_build_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Build Index Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:246" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:247" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-14 16:59:48 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-07-19 08:54:28 +08:00
"description" : "The 99th percentile and average latency of encode index over the last 2 minutes." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-07-19 08:54:28 +08:00
"y" : 14
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123239 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-07-19 08:54:28 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-19 08:54:28 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_indexnode_encode_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_encode_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Encode Index Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:275" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:276" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 16:59:48 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for save index over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-07-19 08:54:28 +08:00
"y" : 14
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123240 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-09-06 18:59:11 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-07-19 08:54:28 +08:00
"legendFormat" : "p99-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
2022-07-19 08:54:28 +08:00
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_indexnode_save_index_latency_sum{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id) / sum(increase(milvus_indexnode_save_index_latency_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])) by(pod, node_id)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
"refId" : "B"
2022-05-09 21:47:53 +08:00
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Save Index Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:331" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:332" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 16:59:48 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Index Node" ,
"type" : "row"
2021-09-16 14:57:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-16 14:57:49 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2022-11-11 14:23:06 +08:00
"y" : 188
2021-09-16 14:57:49 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123157 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
2022-05-23 18:09:59 +08:00
"type" : "prometheus" ,
2022-05-09 21:47:53 +08:00
"uid" : "$datasource"
} ,
"description" : "Total cpu usage of all milvus components." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 183
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123202 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-23 18:09:59 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-23 18:09:59 +08:00
} ,
2022-05-09 21:47:53 +08:00
"exemplar" : true ,
"expr" : "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m]))" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "cpu usage" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "CPU Usage Total" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8411" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8412" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-16 14:57:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 183
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123204 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "memory" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory Total" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:1340" ,
2022-05-09 21:47:53 +08:00
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:1341" ,
2022-05-09 21:47:53 +08:00
"format" : "bytes" ,
"logBase" : 1 ,
"show" : false
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-16 14:57:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of goroutines that currently exist." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-07 10:15:02 +08:00
"y" : 183
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123206 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "goroutines" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Goroutines Total" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:1281" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"label" : "" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-11-07 10:15:02 +08:00
"$$hashKey" : "object:1282" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-16 14:57:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "process cpu usage" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 189
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123135 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "CPU Usage" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-18 16:01:51 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-09 21:47:53 +08:00
} ,
"description" : "Process memory of milvus pod." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 189
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123133 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : false ,
"expr" : "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "bytes" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-18 16:01:51 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of goroutines that currently exist." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-07 10:15:02 +08:00
"y" : 189
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123125 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Goroutines" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-18 16:01:51 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of bytes allocated and still in use." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 195
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123137 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Allocated Memory" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "bytes" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-22 21:55:53 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "GC Max duration seconds." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 195
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123131 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "GC Max duration seconds" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-22 21:55:53 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of OS threads created." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-07 10:15:02 +08:00
"y" : 195
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123127 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "OS Threads" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-22 21:55:53 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Next GC Bytes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 201
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123129 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Next GC Bytes" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-15 09:33:07 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of process opened fds." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 201
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123211 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Process Opened Fds" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-15 09:33:07 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "The fraction of this program's available CPU time used by the GC since the program started." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-07 10:15:02 +08:00
"y" : 201
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123212 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "GC CPU time" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-15 09:33:07 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Rate of memory frees." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 207
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123213 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory Free Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-17 14:43:16 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Rate of go memory mallocs." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 207
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123215 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Memory mallocs rate." ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of allocated heap objects." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-07 10:15:02 +08:00
"y" : 207
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123216 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Allocated Heap Objects" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-17 14:43:16 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
2022-05-23 18:09:59 +08:00
"type" : "prometheus" ,
2022-05-09 21:47:53 +08:00
"uid" : "$datasource"
} ,
"description" : "Rate of bytes allocated, even if freed." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 213
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123217 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-23 18:09:59 +08:00
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-05-23 18:09:59 +08:00
} ,
2022-05-09 21:47:53 +08:00
"exemplar" : true ,
"expr" : "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory Allocated Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8539" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8540" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of heap bytes released to OS." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 213
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123218 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Heap released" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-17 14:43:16 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of heap bytes waiting to be used." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
2022-11-07 10:15:02 +08:00
"y" : 213
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123219 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
2022-08-23 11:50:55 +08:00
"pluginVersion" : "8.3.3" ,
2022-05-09 21:47:53 +08:00
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Heap idle" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2022-08-23 11:50:55 +08:00
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of heap bytes that be used." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
2022-11-07 10:15:02 +08:00
"y" : 219
2022-08-23 11:50:55 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123375 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-08-23 11:50:55 +08:00
} ,
"exemplar" : true ,
"expr" : "go_memstats_heap_inuse_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Heap in Use" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2022-09-06 18:59:11 +08:00
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of container threads created." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
2022-11-07 10:15:02 +08:00
"y" : 219
2022-09-06 18:59:11 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123376 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.3" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2022-09-06 18:59:11 +08:00
} ,
"exemplar" : true ,
"expr" : "container_threads{image=\"\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Container Threads" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-17 14:43:16 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Runtime" ,
"type" : "row"
2021-09-09 17:20:00 +08:00
}
] ,
2022-10-25 15:31:30 +08:00
"refresh" : false ,
2022-08-23 11:50:55 +08:00
"schemaVersion" : 34 ,
2021-09-09 17:20:00 +08:00
"style" : "dark" ,
"tags" : [
"milvus2.0"
] ,
"templating" : {
"list" : [
{
"current" : {
2022-10-19 15:09:28 +08:00
"selected" : false ,
2021-09-09 17:20:00 +08:00
"text" : "prometheus" ,
"value" : "prometheus"
} ,
"hide" : 0 ,
"includeAll" : false ,
"multi" : false ,
"name" : "datasource" ,
"options" : [ ] ,
"query" : "prometheus" ,
"queryValue" : "" ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"type" : "datasource"
} ,
{
"current" : {
2021-11-01 19:23:48 +08:00
"selected" : true ,
2021-09-09 17:20:00 +08:00
"text" : "milvus" ,
"value" : "milvus"
} ,
"hide" : 2 ,
"name" : "app_name" ,
"options" : [
{
2021-10-22 19:53:13 +08:00
"selected" : true ,
2021-09-09 17:20:00 +08:00
"text" : "milvus" ,
"value" : "milvus"
}
] ,
"query" : "milvus" ,
"skipUrlSync" : false ,
"type" : "textbox"
} ,
{
2022-11-07 10:15:02 +08:00
"current" : {
"selected" : true ,
2022-11-11 14:23:06 +08:00
"text" : "chaos-testing" ,
"value" : "chaos-testing"
2022-11-07 10:15:02 +08:00
} ,
2022-05-09 21:47:53 +08:00
"datasource" : {
"uid" : "$datasource"
2021-09-09 17:20:00 +08:00
} ,
"definition" : "label_values(kube_pod_info, namespace)" ,
"hide" : 0 ,
"includeAll" : false ,
"multi" : false ,
"name" : "namespace" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(kube_pod_info, namespace)" ,
"refId" : "prometheus-namespace-Variable-Query"
} ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
2022-11-07 10:15:02 +08:00
"current" : {
"selected" : false ,
2022-11-11 14:23:06 +08:00
"text" : "bulk-insert-test" ,
"value" : "bulk-insert-test"
2022-11-07 10:15:02 +08:00
} ,
2022-05-09 21:47:53 +08:00
"datasource" : {
"uid" : "$datasource"
2021-09-09 17:20:00 +08:00
} ,
"definition" : "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)" ,
"hide" : 0 ,
2021-11-09 16:47:04 +08:00
"includeAll" : false ,
2021-09-09 17:20:00 +08:00
"multi" : false ,
"name" : "instance" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)" ,
2022-10-25 15:31:30 +08:00
"refId" : "StandardVariableQuery"
2021-09-09 17:20:00 +08:00
} ,
"refresh" : 2 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
2022-10-25 15:31:30 +08:00
{
"allValue" : ".*" ,
2022-11-07 10:15:02 +08:00
"current" : {
"selected" : false ,
"text" : "All" ,
"value" : "$__all"
} ,
2022-10-25 15:31:30 +08:00
"datasource" : {
"type" : "prometheus" ,
2023-05-18 14:25:23 +08:00
"uid" : "$datasource"
2022-10-25 15:31:30 +08:00
} ,
"definition" : "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)" ,
"hide" : 0 ,
"includeAll" : true ,
"multi" : false ,
"name" : "collection" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(milvus_proxy_collection_mutation_latency_sum{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"}, collection_name)" ,
"refId" : "StandardVariableQuery"
} ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 0 ,
"type" : "query"
} ,
2021-09-09 17:20:00 +08:00
{
2022-11-07 10:15:02 +08:00
"current" : {
"selected" : false ,
2022-11-11 14:23:06 +08:00
"text" : "bulk-insert-test-milvus-standalone-55968cfc55-cxnps" ,
"value" : "bulk-insert-test-milvus-standalone-55968cfc55-cxnps"
2022-11-07 10:15:02 +08:00
} ,
2022-05-09 21:47:53 +08:00
"datasource" : {
"uid" : "$datasource"
2021-09-09 17:20:00 +08:00
} ,
"definition" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)" ,
"hide" : 2 ,
"includeAll" : false ,
"multi" : true ,
"name" : "pod" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)" ,
"refId" : "prometheus-pod-Variable-Query"
} ,
"refresh" : 2 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
2021-10-19 19:18:34 +08:00
} ,
{
2022-11-07 10:15:02 +08:00
"current" : {
"isNone" : true ,
"selected" : false ,
"text" : "None" ,
"value" : ""
} ,
2022-05-09 21:47:53 +08:00
"datasource" : {
2022-09-23 14:28:51 +08:00
"uid" : "$datasource"
2021-10-19 19:18:34 +08:00
} ,
"definition" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)" ,
"hide" : 2 ,
"includeAll" : false ,
"multi" : true ,
"name" : "component" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)" ,
"refId" : "StandardVariableQuery"
} ,
"refresh" : 2 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
2021-09-09 17:20:00 +08:00
}
]
} ,
"time" : {
2022-11-11 14:23:06 +08:00
"from" : "now-3h" ,
2022-11-04 17:11:36 +08:00
"to" : "now"
2021-09-09 17:20:00 +08:00
} ,
"timepicker" : {
"hidden" : false ,
"refresh_intervals" : [
"5s" ,
"10s" ,
"30s" ,
"1m" ,
"5m" ,
"15m" ,
"30m" ,
"1h" ,
"2h" ,
"1d"
] ,
"time_options" : [
"5m" ,
"15m" ,
"1h" ,
"6h" ,
"12h" ,
"24h" ,
"2d" ,
"7d" ,
"30d"
] ,
"type" : "timepicker"
} ,
"timezone" : "browser" ,
"title" : "Milvus2.0" ,
"uid" : "uLf5cJ3Gz" ,
2022-11-07 10:15:02 +08:00
"version" : 5 ,
2022-05-09 21:47:53 +08:00
"weekStart" : ""
2023-02-10 10:16:31 +08:00
}
