[ADD] add Application instance memory monitoring

This commit is contained in:
zhoujunhao 2018-08-31 15:04:21 +08:00
parent 7064d235d1
commit de7822f7f6
3 changed files with 109 additions and 6 deletions

View File

@ -60,7 +60,7 @@ type ServiceHandler interface {
GetServicesStatus(tenantID string, services []string) map[string]string
CreateTenant(*dbmodel.Tenants) error
CreateTenandIDAndName(eid string) (string, string, error)
GetPods(serviceID string) ([]*dbmodel.K8sPod, error)
GetPods(serviceID string) ([]K8sPodInfo, error)
TransServieToDelete(serviceID string) error
TenantServiceDeletePluginRelation(serviceID, pluginID string) *util.APIHandleError
GetTenantServicePluginRelation(serviceID string) ([]*dbmodel.TenantServicePluginRelation, *util.APIHandleError)

View File

@ -19,6 +19,7 @@
package handler
import (
"context"
"fmt"
"os"
@ -50,6 +51,8 @@ import (
"github.com/Sirupsen/logrus"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1"
"net/http"
"encoding/json"
)
//ServiceAction service act
@ -1764,13 +1767,103 @@ func (s *ServiceAction) CreateTenandIDAndName(eid string) (string, string, error
return uid, name, nil
}
type K8sPodInfo struct {
ServiceID string `json:"service_id"`
//部署资源的ID ,例如rc ,deploment, statefulset
ReplicationID string `json:"rc_id"`
ReplicationType string `json:"rc_type"`
PodName string `json:"pod_name"`
PodIP string `json:"pod_ip"`
Container map[string]map[string]string `json:"container"`
}
//GetPods get pods
func (s *ServiceAction) GetPods(serviceID string) ([]*dbmodel.K8sPod, error) {
func (s *ServiceAction) GetPods(serviceID string) ([]K8sPodInfo, error) {
var podsInfoList []K8sPodInfo
pods, err := db.GetManager().K8sPodDao().GetPodByService(serviceID)
if err != nil {
logrus.Error("GetPodByService Error:", err)
return nil, err
}
return pods, nil
logrus.Info("pods", pods)
for _, v := range pods {
var podInfo K8sPodInfo
containerMemory := make(map[string]map[string]string, 10)
podInfo.ServiceID = v.ServiceID
podInfo.ReplicationID = v.ReplicationID
podInfo.ReplicationType = v.ReplicationType
podInfo.PodName = v.PodName
podInfo.PodIP = v.PodIP
memoryUsageQuery := fmt.Sprintf(`container_memory_usage_bytes{pod_name="%s"}`, v.PodName)
memoryUsageMap, _ := s.GetContainerMemory(memoryUsageQuery)
logrus.Info("memoryUsageMap", memoryUsageMap)
for k, val := range memoryUsageMap {
if _,ok := containerMemory[k];!ok{
containerMemory[k] = map[string]string{"memory_usage": val}
}
}
memorylimitQuery := fmt.Sprintf(`container_spec_memory_limit_bytes{pod_name="%s"}`, v.PodName)
memoryLimitMap, _ := s.GetContainerMemory(memorylimitQuery)
logrus.Info("memoryLimitMap", memoryLimitMap)
for k2, v2 := range memoryLimitMap {
if val, ok := containerMemory[k2]; ok {
val["memory_limit"] = v2
}
}
podInfo.Container = containerMemory
podsInfoList = append(podsInfoList, podInfo)
}
return podsInfoList, nil
}
// Use Prometheus to query memory resources
func (s *ServiceAction) GetContainerMemory(query string) (map[string]string, error) {
memoryUsageMap := make(map[string]string, 10)
proxy := GetPrometheusProxy()
proQuery := strings.Replace(query, " ", "%20", -1)
req, err := http.NewRequest("GET", fmt.Sprintf("http://127.0.0.1:9999/api/v1/query?query=%s", proQuery), nil)
if err != nil {
logrus.Error("create request prometheus api error ", err.Error())
return memoryUsageMap, nil
}
presult, err := proxy.Do(req)
if err != nil {
logrus.Error("do proxy request prometheus api error ", err.Error())
return memoryUsageMap, nil
}
if presult.Body != nil {
defer presult.Body.Close()
if presult.StatusCode != 200 {
logrus.Error("StatusCode:", presult.StatusCode, err)
return memoryUsageMap, nil
}
var qres QueryResult
err = json.NewDecoder(presult.Body).Decode(&qres)
if err == nil {
for _, re := range qres.Data.Result {
var containerName string
var valuesBytes string
if cname, ok := re["metric"].(map[string]interface{}); ok {
containerName = cname["container_name"].(string)
} else {
logrus.Info("metric decode error")
}
if val, ok := (re["value"]).([]interface{}); ok && len(val) == 2 {
valuesBytes = val[1].(string)
} else {
logrus.Info("value decode error")
}
memoryUsageMap[containerName] = valuesBytes
}
return memoryUsageMap, nil
} else {
logrus.Error("Deserialization failed")
}
} else {
logrus.Error("Body Is empty")
}
return memoryUsageMap, nil
}
//TransServieToDelete trans service info to delete table

View File

@ -76,6 +76,16 @@ type serviceInfo struct {
TenantId string `json:"tenantId"`
}
type podInfo struct {
ServiceID string `json:"service_id"`
//部署资源的ID ,例如rc ,deploment, statefulset
ReplicationID string `json:"rc_id"`
ReplicationType string `json:"rc_type"`
PodName string `json:"pod_name"`
PodIP string `json:"pod_ip"`
Container map[string]map[string]string `json:"container"`
}
//NewRegion NewRegion
func NewRegion(c APIConf) (Region, error) {
if region == nil {
@ -195,15 +205,15 @@ func (t *tenant) Services(serviceAlias string) ServiceInterface {
//ServiceInterface ServiceInterface
type ServiceInterface interface {
Get() (*serviceInfo, *util.APIHandleError)
Pods() ([]*dbmodel.K8sPod, *util.APIHandleError)
Pods() ([]*podInfo, *util.APIHandleError)
List() ([]*dbmodel.TenantServices, *util.APIHandleError)
Stop(eventID string) (string, *util.APIHandleError)
Start(eventID string) (string, *util.APIHandleError)
EventLog(eventID, level string) ([]*model.MessageData, *util.APIHandleError)
}
func (s *services) Pods() ([]*dbmodel.K8sPod, *util.APIHandleError) {
var gc []*dbmodel.K8sPod
func (s *services) Pods() ([]*podInfo, *util.APIHandleError) {
var gc []*podInfo
var decode utilhttp.ResponseBody
decode.List = &gc
code, err := s.DoRequest(s.prefix+"/pods", "GET", nil, &decode)