2021-07-23 15:44:12 +08:00
{
2022-05-09 21:47:53 +08:00
"__inputs" : [
{
"name" : "DS_PROMETHEUS" ,
"label" : "prometheus" ,
"description" : "" ,
"type" : "datasource" ,
"pluginId" : "prometheus" ,
"pluginName" : "Prometheus"
}
] ,
"__elements" : [ ] ,
"__requires" : [
{
"type" : "grafana" ,
"id" : "grafana" ,
"name" : "Grafana" ,
"version" : "8.3.2"
} ,
{
"type" : "panel" ,
"id" : "graph" ,
"name" : "Graph (old)" ,
"version" : ""
} ,
{
"type" : "datasource" ,
"id" : "prometheus" ,
"name" : "Prometheus" ,
"version" : "1.0.0"
}
] ,
2021-09-09 17:20:00 +08:00
"annotations" : {
"list" : [
2021-07-23 15:44:12 +08:00
{
2021-09-09 17:20:00 +08:00
"builtIn" : 1 ,
"datasource" : "-- Grafana --" ,
"enable" : true ,
"hide" : true ,
"iconColor" : "rgba(0, 211, 255, 1)" ,
"name" : "Annotations & Alerts" ,
2022-05-09 21:47:53 +08:00
"target" : {
"limit" : 100 ,
"matchAny" : false ,
"tags" : [ ] ,
"type" : "dashboard"
} ,
2021-09-09 17:20:00 +08:00
"type" : "dashboard"
}
]
} ,
"editable" : true ,
2022-05-09 21:47:53 +08:00
"fiscalYearStartMonth" : 0 ,
2021-09-09 17:20:00 +08:00
"graphTooltip" : 0 ,
2022-05-09 21:47:53 +08:00
"id" : null ,
2022-06-10 10:20:07 +08:00
"iteration" : 1654687349785 ,
2021-09-09 17:20:00 +08:00
"links" : [ ] ,
2022-05-09 21:47:53 +08:00
"liveNow" : false ,
2021-09-09 17:20:00 +08:00
"panels" : [
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-09 17:20:00 +08:00
"gridPos" : {
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
2021-12-03 09:21:32 +08:00
"y" : 0
2021-09-09 17:20:00 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123248 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of searching vectors." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 1
} ,
"hiddenSeries" : false ,
"id" : 123250 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_proxy_search_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-05-23 18:09:59 +08:00
"title" : "Search Vector Count Rate" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of inserting vectors." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 1
} ,
"hiddenSeries" : false ,
"id" : 123315 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_proxy_insert_vectors_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-05-23 18:09:59 +08:00
"title" : "Insert Vector Count Rate" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for search request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 1
} ,
"hiddenSeries" : false ,
"id" : 123264 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_lantency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{query_type}}" ,
2022-05-09 21:47:53 +08:00
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-13 19:36:57 +08:00
} ,
2021-09-09 17:34:01 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for waiting search result over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123348 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_wait_result_lantency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}{{query_type}}" ,
2022-05-09 21:47:53 +08:00
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Wait Search Result Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for reducing search result over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123316 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_reduce_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{query_type}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Reduce Search Result Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for decoding search result over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123317 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_proxy_sq_decode_result_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{query_type}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Decode Search Result Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-13 19:36:57 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Average, maximum and minimum values of the number of msgstream objects created on all physical topics." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 13
} ,
"hiddenSeries" : false ,
"id" : 123319 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "avg(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "max(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
2022-05-09 21:47:53 +08:00
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "min(milvus_proxy_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
2022-05-09 21:47:53 +08:00
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Msg Stream Object Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:34:01 +08:00
} ,
2021-09-13 19:36:57 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for mutation request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 13
} ,
"hiddenSeries" : false ,
"id" : 123320 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Mutation Req Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-13 19:36:57 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for sending mutation over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 13
} ,
"hiddenSeries" : false ,
"id" : 123321 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, msg_type, pod, node_id) (rate(milvus_proxy_mutation_send_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Mutation Send Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:15:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Average cache hits per minute of cache operations." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 19
} ,
"hiddenSeries" : false ,
"id" : 123322 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\", cache_state=\"hit\"}[2m])/2) by(cache_name, pod, node_id) / sum(increase(milvus_proxy_cache_hit_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(cache_name, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{cache_name}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Cache Hit rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"max" : "1" ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 16:15:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for updating cache over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 19
} ,
"hiddenSeries" : false ,
"id" : 123323 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_cache_update_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Cache Update Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:34:01 +08:00
} ,
2021-09-13 19:36:57 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Average, maximum and minimum values of the timestamps for all physical channel." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 19
} ,
"hiddenSeries" : false ,
"id" : 123325 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "avg(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-avg" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "max(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-max" ,
2022-05-09 21:47:53 +08:00
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "min(milvus_proxy_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-min" ,
2022-05-09 21:47:53 +08:00
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Sync Time" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 12 ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for applying PK over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 25
} ,
"hiddenSeries" : false ,
"id" : 123326 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_pk_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Apply PK Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-13 19:36:57 +08:00
} ,
2021-09-09 17:34:01 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for applying over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 25
} ,
"hiddenSeries" : false ,
"id" : 123327 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_proxy_apply_timestamp_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Apply Timestamp Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of DQL requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 25
} ,
"hiddenSeries" : false ,
"id" : 123328 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_proxy_dql_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(function_name, status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{function_name}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DQL Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-07-23 15:44:12 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of DML requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 31
} ,
"hiddenSeries" : false ,
"id" : 123329 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_proxy_dml_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(function_name, status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{function_name}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DML Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of DDL requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 31
} ,
"hiddenSeries" : false ,
"id" : 123330 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_proxy_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by(function_name, status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{function_name}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DDL Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-13 19:36:57 +08:00
} ,
2021-09-09 17:34:01 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for DQL request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 31
} ,
"hiddenSeries" : false ,
"id" : 123331 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, function_name, pod, node_id) (rate(milvus_proxy_dql_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{function_name}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DQL Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for DML request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 37
} ,
"hiddenSeries" : false ,
"id" : 123332 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, function_name, pod, node_id) (rate(milvus_proxy_dml_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{function_name}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DML Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-15 17:15:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for DDL request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 37
} ,
"hiddenSeries" : false ,
"id" : 123333 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, function_name, pod, node_id) (rate(milvus_proxy_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{function_name}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DDL Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3538" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3539" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-15 17:15:49 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Proxy" ,
"type" : "row"
2021-09-15 17:15:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-15 17:15:49 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
2021-12-07 16:15:03 +08:00
"x" : 0 ,
2022-05-09 21:47:53 +08:00
"y" : 1
2021-09-15 17:15:49 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123155 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of proxy nodes which has register with etcd" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 2
} ,
"hiddenSeries" : false ,
"id" : 123141 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_proxy_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Proxy Node Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : " Timestamps for each physical channel" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 2
} ,
"hiddenSeries" : false ,
"id" : 123335 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "avg(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "avg time tick" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "max(milvus_rootcoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "max time tick" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "min(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "min time tick" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Sync Time" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 12 ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-15 17:15:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of DDL request" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 2
} ,
"hiddenSeries" : false ,
"id" : 123347 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_rootcoord_ddl_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, function_name)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{function_name}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DDL Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:1456" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:1457" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for DDL request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 8
} ,
"hiddenSeries" : false ,
"id" : 123337 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "{{function_name}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DDL Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-15 17:15:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for rootcoord to finish synchronizing timestamp messages to all pchanels." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 8
} ,
"hiddenSeries" : false ,
"id" : 123338 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "latency" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Sync Timetick Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of IDs assigned by RootCoord" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 8
} ,
"hiddenSeries" : false ,
"id" : 123221 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_rootcoord_id_alloc_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2)" ,
"interval" : "" ,
"legendFormat" : "total" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "ID Alloc Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:1456" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:1457" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "RoootCoord current latest timestamp" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 14
} ,
"hiddenSeries" : false ,
"id" : 123339 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "milvus_rootcoord_timestamp{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "tiestamp" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Timestamp" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 16:43:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "RootCoord stores pre-assigned timestamps in the metastore" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 14
} ,
"hiddenSeries" : false ,
"id" : 123340 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "milvus_rootcoord_timestamp_saved{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "tiestamp" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Timestamp Saved" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of collections." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 14
} ,
"hiddenSeries" : false ,
"id" : 123342 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of partitions." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 20
} ,
"hiddenSeries" : false ,
"id" : 123344 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Partition Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of DML channels" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 20
} ,
"hiddenSeries" : false ,
"id" : 123345 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_dml_channel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DML Channel Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 16:43:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of Msgstream objects." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 20
} ,
"hiddenSeries" : false ,
"id" : 123346 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_msgstream_obj_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Msgstream Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of credential" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 26
} ,
"hiddenSeries" : false ,
"id" : 123343 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_rootcoord_credential_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Credential Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:4353" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:4354" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Root Coordinator" ,
"type" : "row"
2021-09-15 17:15:49 +08:00
} ,
2021-12-07 14:57:03 +08:00
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-12-07 14:57:03 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
2021-12-07 16:43:03 +08:00
"x" : 0 ,
2022-05-09 21:47:53 +08:00
"y" : 2
} ,
"id" : 123246 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of loaded collections." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 3
} ,
"hiddenSeries" : false ,
"id" : 123288 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querycoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 14:57:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of loaded entities." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 3
} ,
"hiddenSeries" : false ,
"id" : 123289 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-06-10 10:20:07 +08:00
"expr" : "sum(milvus_querycoord_entity_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Entity Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of load requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 3
} ,
"hiddenSeries" : false ,
"id" : 123291 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_querycoord_load_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status)" ,
"interval" : "" ,
"legendFormat" : "{{status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Load Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 16:43:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of relaase requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 9
} ,
"hiddenSeries" : false ,
"id" : 123292 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_querycoord_release_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status)" ,
"interval" : "" ,
"legendFormat" : "{{status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Release Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for load request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 9
} ,
"hiddenSeries" : false ,
"id" : 123294 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_load_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "latency" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Load Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 14:57:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for DQL release over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 9
} ,
"hiddenSeries" : false ,
"id" : 123313 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_release_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "latency" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Release Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of sub-load task." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 15
} ,
"hiddenSeries" : false ,
"id" : 123295 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querycoord_child_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Sub-Load Task" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number pf parent loading task." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 15
} ,
"hiddenSeries" : false ,
"id" : 123296 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querycoord_parent_task_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Parent Load Task" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 16:43:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for sub-load task request over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 15
} ,
"hiddenSeries" : false ,
"id" : 123298 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le) (rate(milvus_querycoord_child_task_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "latency" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Sub-Load Task Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of Query nodes which has register with etcd." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 21
} ,
"hiddenSeries" : false ,
"id" : 123297 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querycoord_querynode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "num" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Query Node Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-07 16:43:03 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Query Coordinator" ,
"type" : "row"
2021-12-07 14:57:03 +08:00
} ,
2021-09-15 17:15:49 +08:00
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-15 17:15:49 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 3
} ,
"id" : 123244 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of the loaded collections in QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 4
} ,
"hiddenSeries" : false ,
"id" : 123299 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_querynode_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-15 17:15:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of the loaded partitions in QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 4
} ,
"hiddenSeries" : false ,
"id" : 123303 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_querynode_partition_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Partition Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-07 16:43:03 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of the loaded segment in QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 4
} ,
"hiddenSeries" : false ,
"id" : 123305 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_querynode_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Segment Loaded Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 0 ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-15 17:15:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of dml virtual channels for QueryNode watch" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 10
} ,
"hiddenSeries" : false ,
"id" : 123304 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_querynode_dml_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "DML Virtual Channel" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-07-23 15:44:12 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of delta virtual channels for QueryNode watch" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 10
} ,
"hiddenSeries" : false ,
"id" : 123306 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_querynode_delta_vchannel_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Delta Virtual Channel" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-07-23 15:44:12 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of consumers in the QueryNode." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 10
} ,
"hiddenSeries" : false ,
"id" : 123307 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_querynode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Consumer Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-07-23 15:44:12 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
2022-05-31 09:02:03 +08:00
"description" : "per-minute increasing rate of searching requests." ,
2022-05-09 21:47:53 +08:00
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 16
} ,
"hiddenSeries" : false ,
2022-05-31 09:02:03 +08:00
"id" : 123350 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_sq_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (query_type, status, pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}-{{query_type}}-{{status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for searching segment over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 16
} ,
"hiddenSeries" : false ,
2022-05-09 21:47:53 +08:00
"id" : 123300 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : false ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, query_type, segment_state, pod, node_id) (rate(milvus_querynode_sq_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"instant" : true ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{query_type}}-{{segment_state}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Segment Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-11-01 19:23:48 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for searching at the segcore step over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-05-31 09:02:03 +08:00
"x" : 16 ,
2022-05-09 21:47:53 +08:00
"y" : 16
} ,
"hiddenSeries" : false ,
"id" : 123310 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, query_type, pod, node_id) (rate(milvus_querynode_sq_core_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{query_type}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Segcore Request Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-07-23 15:44:12 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for loading segment over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-05-31 09:02:03 +08:00
"x" : 0 ,
"y" : 22
2022-05-09 21:47:53 +08:00
} ,
"hiddenSeries" : false ,
"id" : 123311 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_load_segment_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Load Segment Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of flowgraph" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
2022-05-31 09:02:03 +08:00
"x" : 8 ,
2022-05-09 21:47:53 +08:00
"y" : 22
} ,
"hiddenSeries" : false ,
"id" : 123312 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_querynode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Flowgraph Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2022-06-10 10:20:07 +08:00
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The length of the task queue for unsolved read requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 22
} ,
"hiddenSeries" : false ,
"id" : 123351 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querynode_read_task_unsolved_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Unsolved Read Task Length" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The length of the task queue of read requests to be executed" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 28
} ,
"hiddenSeries" : false ,
"id" : 123356 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querynode_read_task_ready_len{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Ready Read Task Length" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of read requests currently being executed in parallel" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 28
} ,
"hiddenSeries" : false ,
"id" : 123357 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querynode_read_task_concurrency{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Parallel Read Task Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "cpu utilization under scheduler evaluation" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 28
} ,
"hiddenSeries" : false ,
"id" : 123358 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_querynode_estimate_cpu_usage{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Estimate CPU Usage" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of original tasks contained in the merged search tas" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 34
} ,
"hiddenSeries" : false ,
"id" : 123352 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_size_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Group Size" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of queries for search requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 34
} ,
"hiddenSeries" : false ,
"id" : 123361 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search NQ" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of queries for the merged search requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 34
} ,
"hiddenSeries" : false ,
"id" : 123360 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_nq_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Group NQ" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Top_K for search requests" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 40
} ,
"hiddenSeries" : false ,
"id" : 123359 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Top_K" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Top_K for the merged search requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 40
} ,
"hiddenSeries" : false ,
"id" : 123362 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_querynode_search_group_topk_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Search Group Top_K" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of evicted read requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 40
} ,
"hiddenSeries" : false ,
"id" : 123364 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_querynode_read_evicted_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (pod, node_id)" ,
"interval" : "" ,
"legendFormat" : "{{pod}}-{{node_id}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Evicted Read Requests Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:3414" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:3415" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-07-23 15:44:12 +08:00
}
2021-09-09 17:20:00 +08:00
] ,
2022-05-09 21:47:53 +08:00
"title" : "Query Node" ,
"type" : "row"
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-09 17:20:00 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
2021-09-09 17:20:00 +08:00
"x" : 0 ,
2022-05-09 21:47:53 +08:00
"y" : 4
2021-09-09 17:20:00 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123172 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of data nodes which has register with etcd." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 5
} ,
"hiddenSeries" : false ,
"id" : 123207 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_datanode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "total" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Data Node Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of segments with different states in the Meta of DataCoord" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 5
} ,
"hiddenSeries" : false ,
"id" : 123267 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (segment_state)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{segment_state}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Segment Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of collections recorded in DataCoord Meta" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 5
} ,
"hiddenSeries" : false ,
"id" : 123268 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_collection_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "total" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Collection Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The number of rows of valid data accumulated in DataCoord that flushed." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 11
} ,
"hiddenSeries" : false ,
"id" : 123269 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_datacoord_stored_rows_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "row" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Stored Rows" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "On the DataCoord, the average, max and min timestmp on each physical topic" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 11
} ,
"hiddenSeries" : false ,
"id" : 123270 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "avg(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "avg time tick" ,
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "max(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "max time tick" ,
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
"expr" : "min(milvus_datacoord_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"hide" : false ,
"interval" : "" ,
"legendFormat" : "min time tick" ,
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Sync Time" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"decimals" : 12 ,
"format" : "short" ,
"label" : "" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Data Coordinator" ,
"type" : "row"
2021-07-23 15:44:12 +08:00
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-09 17:20:00 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 5
} ,
"id" : 123242 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of flowgraph" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 6
} ,
"hiddenSeries" : false ,
"id" : 123272 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_datanode_flowgraph_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Flowgraph Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of messages consumed for insert and delete operation." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 6
} ,
"hiddenSeries" : false ,
"id" : 123274 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_datanode_msg_rows_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (msg_type, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Msg Rows Consum Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of each message that has been flushed." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 6
} ,
"hiddenSeries" : false ,
"id" : 123275 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_datanode_flushed_data_size{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (msg_type, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{msg_type}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Flush Data Size Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of consumers created on the DataNode" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 12
} ,
"hiddenSeries" : false ,
"id" : 123276 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_datanode_consumer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Consumer Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of prodecer created on the DataNode" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 12
} ,
"hiddenSeries" : false ,
"id" : 123277 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_datanode_producer_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Producer Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "On the DataNode, the average, max and min timestmp on each physical topic" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 12
} ,
"hiddenSeries" : false ,
"id" : 123279 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "avg(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "avg-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "max(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "max-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"refId" : "B"
} ,
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "min(milvus_datanode_sync_epoch_time{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"hide" : false ,
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "min-{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"refId" : "C"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Sync Time" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Total number of segment that has been not flushed." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 18
} ,
"hiddenSeries" : false ,
"id" : 123280 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(milvus_datanode_unflushed_segment_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
"intervalFactor" : 2 ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Unflushed Segment Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:536" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:537" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for encoding the data in the buffer over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 18
} ,
"hiddenSeries" : false ,
"id" : 123282 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_encode_buffer_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Encode Buffer Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-09 17:20:00 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for writting the data in buffer to storage over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 18
} ,
"hiddenSeries" : false ,
"id" : 123283 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_save_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Save Data Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-09 17:20:00 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of flush operete." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 24
} ,
"hiddenSeries" : false ,
"id" : 123284 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_datanode_flush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Flush Operate Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-14 14:53:48 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of auto flush operate." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 24
} ,
"hiddenSeries" : false ,
"id" : 123285 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_datanode_autoflush_buffer_op_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Autoflush Operate Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-14 14:53:48 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of flush requests." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 24
} ,
"hiddenSeries" : false ,
"id" : 123286 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_datanode_flush_req_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Flush Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 14:53:48 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for compaction over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 30
} ,
"hiddenSeries" : false ,
"id" : 123314 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_datanode_compaction_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Compaction Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 14:53:48 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Data Node" ,
"type" : "row"
2021-09-14 15:57:47 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-14 15:57:47 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 6
} ,
"id" : 123223 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of build requests recived by IndexCoord." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123225 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(increase(milvus_indexcoord_indexreq_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status)" ,
"interval" : "" ,
"legendFormat" : "{{status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Index Request Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:372" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:373" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-14 15:57:47 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of indexing tasks with different states in IndexCoord's meta" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123227 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_indexcoord_indextask_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (index_task_status)" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{index_task_status}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Index Task Count" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:401" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:402" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 15:57:47 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "Number of index nodes which has register with etcd" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 7
} ,
"hiddenSeries" : false ,
"id" : 123229 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
"expr" : "sum(milvus_indexcoord_indexnode_num{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}) by (app_kubernetes_io_instance)" ,
"hide" : false ,
"instant" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "total" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Index Node Num" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:479" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:480" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 15:57:47 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Index Coordinator" ,
"type" : "row"
2021-09-14 16:59:48 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-14 16:59:48 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
2021-09-14 16:59:48 +08:00
"x" : 0 ,
2022-05-09 21:47:53 +08:00
"y" : 7
2021-09-14 16:59:48 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123231 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "per-minute increasing rate of index tasks recevied by IndexNode" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 8
} ,
"hiddenSeries" : false ,
"id" : 123233 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "sum(increase(milvus_indexnode_index_task_count{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])/2) by (status, pod, node_id)" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}-{{status}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
2022-05-23 18:09:59 +08:00
"title" : "Index Task Rate" ,
2022-05-09 21:47:53 +08:00
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:101" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:102" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-14 16:59:48 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for loading the FieldData over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 8
} ,
"hiddenSeries" : false ,
"id" : 123235 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_load_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Load Field Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:161" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:162" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for decoding the FieldData over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 8
} ,
"hiddenSeries" : false ,
"id" : 123238 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_decode_field_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Decode Field Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:190" ,
"format" : "short" ,
"label" : "" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:191" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for building index over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 14
} ,
"hiddenSeries" : false ,
"id" : 123237 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_build_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Build Index Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:246" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:247" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-14 16:59:48 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for encoding index over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 14
} ,
"hiddenSeries" : false ,
"id" : 123239 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_encode_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Encode Index Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:275" ,
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"$$hashKey" : "object:276" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 16:59:48 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "$datasource"
} ,
"description" : "The 99th percentile of latency for save index over the last 2 minutes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 14
} ,
"hiddenSeries" : false ,
"id" : 123240 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"exemplar" : true ,
2022-05-23 18:09:59 +08:00
"expr" : "histogram_quantile(0.99, sum by (le, pod, node_id) (rate(milvus_indexnode_save_index_latency_bucket{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}[2m])))" ,
2022-05-09 21:47:53 +08:00
"interval" : "" ,
2022-05-23 18:09:59 +08:00
"legendFormat" : "{{pod}}-{{node_id}}" ,
2022-05-09 21:47:53 +08:00
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Save Index Latency" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"$$hashKey" : "object:331" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"$$hashKey" : "object:332" ,
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-14 16:59:48 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Index Node" ,
"type" : "row"
2021-09-16 14:57:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"collapsed" : true ,
2021-09-16 14:57:49 +08:00
"gridPos" : {
2022-05-09 21:47:53 +08:00
"h" : 1 ,
"w" : 24 ,
"x" : 0 ,
"y" : 8
2021-09-16 14:57:49 +08:00
} ,
2022-05-09 21:47:53 +08:00
"id" : 123157 ,
"panels" : [
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
2022-05-23 18:09:59 +08:00
"type" : "prometheus" ,
2022-05-09 21:47:53 +08:00
"uid" : "$datasource"
} ,
"description" : "Total cpu usage of all milvus components." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 9
} ,
"hiddenSeries" : false ,
"id" : 123202 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-23 18:09:59 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
2022-05-09 21:47:53 +08:00
"exemplar" : true ,
"expr" : "sum(rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m]))" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "cpu usage" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "CPU Usage Total" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8411" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8412" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-16 14:57:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 9
} ,
"hiddenSeries" : false ,
"id" : 123204 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "sum(process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "memory" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory Total" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "bytes" ,
"logBase" : 1 ,
"show" : false
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-16 14:57:49 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of goroutines that currently exist." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 9
} ,
"hiddenSeries" : false ,
"id" : 123206 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "sum(go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"})" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "goroutines" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Goroutines Total" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"label" : "" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-16 14:57:49 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "process cpu usage" ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 15
} ,
"hiddenSeries" : false ,
"id" : 123135 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "rate(process_cpu_seconds_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "CPU Usage" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-18 16:01:51 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
"description" : "Process memory of milvus pod." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 15
} ,
"hiddenSeries" : false ,
"id" : 123133 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : false ,
"expr" : "process_resident_memory_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"hide" : false ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "bytes" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-18 16:01:51 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of goroutines that currently exist." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 15
} ,
"hiddenSeries" : false ,
"id" : 123125 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_goroutines{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Goroutines" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-18 16:01:51 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of bytes allocated and still in use." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 21
} ,
"hiddenSeries" : false ,
"id" : 123137 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_alloc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Allocated Memory" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "bytes" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-22 21:55:53 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "GC Max duration seconds." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 21
} ,
"hiddenSeries" : false ,
"id" : 123131 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_gc_duration_seconds{quantile=\"1\", app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "GC Max duration seconds" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-09-22 21:55:53 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of OS threads created." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 21
} ,
"hiddenSeries" : false ,
"id" : 123127 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_threads{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "OS Threads" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-09-22 21:55:53 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Next GC Bytes." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 27
} ,
"hiddenSeries" : false ,
"id" : 123129 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_next_gc_bytes{app_kubernetes_io_instance=~\"$instance\", app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Next GC Bytes" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-15 09:33:07 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of process opened fds." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 27
} ,
"hiddenSeries" : false ,
"id" : 123211 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "process_open_fds{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Process Opened Fds" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-15 09:33:07 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "The fraction of this program's available CPU time used by the GC since the program started." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 27
} ,
"hiddenSeries" : false ,
"id" : 123212 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_gc_cpu_fraction{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "GC CPU time" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-15 09:33:07 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Rate of memory frees." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 33
} ,
"hiddenSeries" : false ,
"id" : 123213 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "rate(go_memstats_frees_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory Free Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-17 14:43:16 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Rate of go memory mallocs." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 33
} ,
"hiddenSeries" : false ,
"id" : 123215 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "rate(go_memstats_mallocs_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[2m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Memory mallocs rate." ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of allocated heap objects." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 33
} ,
"hiddenSeries" : false ,
"id" : 123216 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_heap_objects{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Allocated Heap Objects" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
2021-12-17 14:43:16 +08:00
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
2022-05-23 18:09:59 +08:00
"type" : "prometheus" ,
2022-05-09 21:47:53 +08:00
"uid" : "$datasource"
} ,
"description" : "Rate of bytes allocated, even if freed." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 0 ,
"y" : 39
} ,
"hiddenSeries" : false ,
"id" : 123217 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
2022-05-23 18:09:59 +08:00
"datasource" : {
"type" : "prometheus" ,
"uid" : "${DS_PROMETHEUS}"
} ,
2022-05-09 21:47:53 +08:00
"exemplar" : true ,
"expr" : "rate(go_memstats_alloc_bytes_total{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}[5m])" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Memory Allocated Rate" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8539" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
2022-05-23 18:09:59 +08:00
"$$hashKey" : "object:8540" ,
2022-05-09 21:47:53 +08:00
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
} ,
{
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of heap bytes released to OS." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 8 ,
"y" : 39
} ,
"hiddenSeries" : false ,
"id" : 123218 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_heap_released_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Heap released" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-17 14:43:16 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"aliasColors" : { } ,
"bars" : false ,
"dashLength" : 10 ,
"dashes" : false ,
"datasource" : {
"uid" : "$datasource"
} ,
"description" : "Number of heap bytes waiting to be used." ,
"fill" : 1 ,
"fillGradient" : 0 ,
"gridPos" : {
"h" : 6 ,
"w" : 8 ,
"x" : 16 ,
"y" : 39
} ,
"hiddenSeries" : false ,
"id" : 123219 ,
"legend" : {
"avg" : false ,
"current" : false ,
"max" : false ,
"min" : false ,
"show" : true ,
"total" : false ,
"values" : false
} ,
"lines" : true ,
"linewidth" : 1 ,
"nullPointMode" : "null" ,
"options" : {
"alertThreshold" : true
} ,
"percentage" : false ,
"pluginVersion" : "8.3.2" ,
"pointradius" : 2 ,
"points" : false ,
"renderer" : "flot" ,
"seriesOverrides" : [ ] ,
"spaceLength" : 10 ,
"stack" : false ,
"steppedLine" : false ,
"targets" : [
{
"exemplar" : true ,
"expr" : "go_memstats_heap_idle_bytes{app_kubernetes_io_name=\"$app_name\", app_kubernetes_io_instance=~\"$instance\", namespace=\"$namespace\"}" ,
"interval" : "" ,
"intervalFactor" : 2 ,
"legendFormat" : "{{pod}}" ,
"queryType" : "randomWalk" ,
"refId" : "A"
}
] ,
"thresholds" : [ ] ,
"timeRegions" : [ ] ,
"title" : "Go Heap idle" ,
"tooltip" : {
"shared" : true ,
"sort" : 0 ,
"value_type" : "individual"
} ,
"type" : "graph" ,
"xaxis" : {
"mode" : "time" ,
"show" : true ,
"values" : [ ]
} ,
"yaxes" : [
{
"format" : "bytes" ,
"logBase" : 1 ,
"min" : "0" ,
"show" : true
} ,
{
"format" : "short" ,
"logBase" : 1 ,
"show" : true
}
] ,
"yaxis" : {
"align" : false
}
2021-12-17 14:43:16 +08:00
}
] ,
2022-05-09 21:47:53 +08:00
"title" : "Runtime" ,
"type" : "row"
2021-09-09 17:20:00 +08:00
}
] ,
2021-09-13 15:28:10 +08:00
"refresh" : false ,
2022-05-09 21:47:53 +08:00
"schemaVersion" : 33 ,
2021-09-09 17:20:00 +08:00
"style" : "dark" ,
"tags" : [
"milvus2.0"
] ,
"templating" : {
"list" : [
{
"current" : {
"selected" : false ,
"text" : "prometheus" ,
"value" : "prometheus"
} ,
"hide" : 0 ,
"includeAll" : false ,
"multi" : false ,
"name" : "datasource" ,
"options" : [ ] ,
"query" : "prometheus" ,
"queryValue" : "" ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"type" : "datasource"
} ,
{
"current" : {
2021-11-01 19:23:48 +08:00
"selected" : true ,
2021-09-09 17:20:00 +08:00
"text" : "milvus" ,
"value" : "milvus"
} ,
"hide" : 2 ,
"name" : "app_name" ,
"options" : [
{
2021-10-22 19:53:13 +08:00
"selected" : true ,
2021-09-09 17:20:00 +08:00
"text" : "milvus" ,
"value" : "milvus"
}
] ,
"query" : "milvus" ,
"skipUrlSync" : false ,
"type" : "textbox"
} ,
{
2022-05-09 21:47:53 +08:00
"current" : { } ,
"datasource" : {
"uid" : "$datasource"
2021-09-09 17:20:00 +08:00
} ,
"definition" : "label_values(kube_pod_info, namespace)" ,
"hide" : 0 ,
"includeAll" : false ,
"multi" : false ,
"name" : "namespace" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(kube_pod_info, namespace)" ,
"refId" : "prometheus-namespace-Variable-Query"
} ,
"refresh" : 1 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
2022-05-09 21:47:53 +08:00
"current" : { } ,
"datasource" : {
"uid" : "$datasource"
2021-09-09 17:20:00 +08:00
} ,
"definition" : "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)" ,
"hide" : 0 ,
2021-11-09 16:47:04 +08:00
"includeAll" : false ,
2021-09-09 17:20:00 +08:00
"multi" : false ,
"name" : "instance" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(go_info{app_kubernetes_io_name=\"$app_name\", namespace=\"$namespace\"}, app_kubernetes_io_instance)" ,
"refId" : "prometheus-instance-Variable-Query"
} ,
"refresh" : 2 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
} ,
{
2022-05-09 21:47:53 +08:00
"current" : { } ,
"datasource" : {
"uid" : "$datasource"
2021-09-09 17:20:00 +08:00
} ,
"definition" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)" ,
"hide" : 2 ,
"includeAll" : false ,
"multi" : true ,
"name" : "pod" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , pod)" ,
"refId" : "prometheus-pod-Variable-Query"
} ,
"refresh" : 2 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
2021-10-19 19:18:34 +08:00
} ,
{
2022-05-09 21:47:53 +08:00
"current" : { } ,
"datasource" : {
"uid" : "${DS_PROMETHEUS}"
2021-10-19 19:18:34 +08:00
} ,
"definition" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)" ,
"hide" : 2 ,
"includeAll" : false ,
"multi" : true ,
"name" : "component" ,
"options" : [ ] ,
"query" : {
"query" : "label_values(go_info{namespace=\"$namespace\", app_kubernetes_io_instance=~\"$instance\",app_kubernetes_io_name=\"$app_name\"} , app_kubernetes_io_component)" ,
"refId" : "StandardVariableQuery"
} ,
"refresh" : 2 ,
"regex" : "" ,
"skipUrlSync" : false ,
"sort" : 1 ,
"tagValuesQuery" : "" ,
"tagsQuery" : "" ,
"type" : "query" ,
"useTags" : false
2021-09-09 17:20:00 +08:00
}
]
} ,
"time" : {
2022-05-09 21:47:53 +08:00
"from" : "now-5m" ,
2021-09-13 17:58:58 +08:00
"to" : "now"
2021-09-09 17:20:00 +08:00
} ,
"timepicker" : {
"hidden" : false ,
"refresh_intervals" : [
"5s" ,
"10s" ,
"30s" ,
"1m" ,
"5m" ,
"15m" ,
"30m" ,
"1h" ,
"2h" ,
"1d"
] ,
"time_options" : [
"5m" ,
"15m" ,
"1h" ,
"6h" ,
"12h" ,
"24h" ,
"2d" ,
"7d" ,
"30d"
] ,
"type" : "timepicker"
} ,
"timezone" : "browser" ,
"title" : "Milvus2.0" ,
"uid" : "uLf5cJ3Gz" ,
2022-06-10 10:20:07 +08:00
"version" : 5 ,
2022-05-09 21:47:53 +08:00
"weekStart" : ""
2022-05-23 18:09:59 +08:00
}
