[ADD] record OOMKilled

db/dao/dao.go

@@ -312,7 +312,8 @@ type EventDao interface {
 	DelEventByServiceID(serviceID string) error
 	GetEventsByTarget(target, targetID string, offset, liimt int) ([]*model.ServiceEvent, int, error)
 	GetEventsByTenantID(tenantID string, offset, limit int) ([]*model.ServiceEvent, int, error)
-	GetBySIDAndType(serviceID string, optTypes ...string) (*model.ServiceEvent, error)
+	GetByTargetIDTypeUser(targetID, optType, username string) (*model.ServiceEvent, error)
+	GetByTargetIDAndType(targetID string, optTypes ...string) (*model.ServiceEvent, error)
 	GetLastASyncEvent(target, targetID string) (*model.ServiceEvent, error)
 }
 

db/db.go

@@ -25,7 +25,6 @@ import (
 	"github.com/goodrain/rainbond/db/config"
 	"github.com/goodrain/rainbond/db/dao"
 	"github.com/goodrain/rainbond/db/mysql"
-
 	"github.com/jinzhu/gorm"
 )
 
@@ -144,3 +143,8 @@ func CloseManager() error {
 func GetManager() Manager {
 	return defaultManager
 }
+
+// SetManager sets the default manager, usally for unit test
+func SetManager(m Manager) {
+	defaultManager = m
+}

db/model/event.go

@@ -32,6 +32,26 @@ const TargetTypeService = "service"
 // TargetTypeTenant tenant target
 const TargetTypeTenant = "tenant"
 
+// UsernameSystem -
+const UsernameSystem = "system"
+
+// EventFinalStatus -
+type EventFinalStatus string
+
+// String -
+func (e EventFinalStatus) String() string {
+	return string(e)
+}
+
+// EventFinalStatusComplete -
+var EventFinalStatusComplete EventFinalStatus = "complete"
+
+// EventFinalStatusFailure -
+var EventFinalStatusFailure EventFinalStatus = "failure"
+
+// EventFinalStatusRunning -
+var EventFinalStatusRunning EventFinalStatus = "running"
+
 //ServiceEvent event struct
 type ServiceEvent struct {
 	Model

db/mysql/dao/event.go

@@ -149,10 +149,19 @@ func (c *EventDaoImpl) GetEventsByTenantID(tenantID string, offset, limit int) (
 	return result, total, nil
 }
 
-// GetBySIDAndType -
-func (c *EventDaoImpl) GetBySIDAndType(serviceID string, optTypes ...string) (*model.ServiceEvent, error) {
+// GetByTargetIDTypeUser -
+func (c *EventDaoImpl) GetByTargetIDTypeUser(targetID, optType, username string) (*model.ServiceEvent, error) {
 	var result model.ServiceEvent
-	if err := c.DB.Where("service_id=? and opt_type in (?)", serviceID, optTypes).Last(&result).Error; err != nil {
+	if err := c.DB.Where("target_id=? and opt_type=? and user_name=?", targetID, optType, username).Last(&result).Error; err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// GetByTargetIDAndType -
+func (c *EventDaoImpl) GetByTargetIDAndType(targetID string, optTypes ...string) (*model.ServiceEvent, error) {
+	var result model.ServiceEvent
+	if err := c.DB.Where("target_id=? and opt_type in (?)", targetID, optTypes).Last(&result).Error; err != nil {
 		return nil, err
 	}
 	return &result, nil

event/manager.go

@@ -26,17 +26,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/goodrain/rainbond/discover"
 	"github.com/goodrain/rainbond/discover/config"
-	"github.com/goodrain/rainbond/util"
-
-	"github.com/pquerna/ffjson/ffjson"
-
-	"github.com/Sirupsen/logrus"
-
 	eventclient "github.com/goodrain/rainbond/eventlog/entry/grpc/client"
 	eventpb "github.com/goodrain/rainbond/eventlog/entry/grpc/pb"
-
+	"github.com/goodrain/rainbond/util"
+	"github.com/pquerna/ffjson/ffjson"
 	"golang.org/x/net/context"
 )
 
@@ -105,6 +101,11 @@ func GetManager() Manager {
 	return defaultManager
 }
 
+// SetManager -
+func SetManager(m Manager) {
+	defaultManager = m
+}
+
 //CloseManager 关闭日志服务
 func CloseManager() {
 	if defaultManager != nil {
@@ -217,11 +218,7 @@ func (m *manager) GetLogger(eventID string) Logger {
 	if l, ok := m.loggers[eventID]; ok {
 		return l
 	}
-	l := &logger{
-		event:      eventID,
-		sendChan:   m.getLBChan(),
-		createTime: time.Now(),
-	}
+	l := NewLogger(eventID, m.getLBChan())
 	m.loggers[eventID] = l
 	return l
 }
@@ -350,6 +347,15 @@ type Logger interface {
 	GetWriter(step, level string) LoggerWriter
 }
 
+// NewLogger creates a new Logger.
+func NewLogger(eventID string, sendCh chan []byte) Logger {
+	return &logger{
+		event:      eventID,
+		sendChan:   sendCh,
+		createTime: time.Now(),
+	}
+}
+
 type logger struct {
 	event      string
 	sendChan   chan []byte

event/manager_mock.go

@@ -0,0 +1,265 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: event/manager.go
+
+// Package event is a generated GoMock package.
+package event
+
+import (
+	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
+	time "time"
+)
+
+// MockManager is a mock of Manager interface
+type MockManager struct {
+	ctrl     *gomock.Controller
+	recorder *MockManagerMockRecorder
+}
+
+// MockManagerMockRecorder is the mock recorder for MockManager
+type MockManagerMockRecorder struct {
+	mock *MockManager
+}
+
+// NewMockManager creates a new mock instance
+func NewMockManager(ctrl *gomock.Controller) *MockManager {
+	mock := &MockManager{ctrl: ctrl}
+	mock.recorder = &MockManagerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockManager) EXPECT() *MockManagerMockRecorder {
+	return m.recorder
+}
+
+// GetLogger mocks base method
+func (m *MockManager) GetLogger(eventID string) Logger {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetLogger", eventID)
+	ret0, _ := ret[0].(Logger)
+	return ret0
+}
+
+// GetLogger indicates an expected call of GetLogger
+func (mr *MockManagerMockRecorder) GetLogger(eventID interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLogger", reflect.TypeOf((*MockManager)(nil).GetLogger), eventID)
+}
+
+// Start mocks base method
+func (m *MockManager) Start() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Start")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Start indicates an expected call of Start
+func (mr *MockManagerMockRecorder) Start() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockManager)(nil).Start))
+}
+
+// Close mocks base method
+func (m *MockManager) Close() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close
+func (mr *MockManagerMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockManager)(nil).Close))
+}
+
+// ReleaseLogger mocks base method
+func (m *MockManager) ReleaseLogger(arg0 Logger) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "ReleaseLogger", arg0)
+}
+
+// ReleaseLogger indicates an expected call of ReleaseLogger
+func (mr *MockManagerMockRecorder) ReleaseLogger(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseLogger", reflect.TypeOf((*MockManager)(nil).ReleaseLogger), arg0)
+}
+
+// MockLogger is a mock of Logger interface
+type MockLogger struct {
+	ctrl     *gomock.Controller
+	recorder *MockLoggerMockRecorder
+}
+
+// MockLoggerMockRecorder is the mock recorder for MockLogger
+type MockLoggerMockRecorder struct {
+	mock *MockLogger
+}
+
+// NewMockLogger creates a new mock instance
+func NewMockLogger(ctrl *gomock.Controller) *MockLogger {
+	mock := &MockLogger{ctrl: ctrl}
+	mock.recorder = &MockLoggerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockLogger) EXPECT() *MockLoggerMockRecorder {
+	return m.recorder
+}
+
+// Info mocks base method
+func (m *MockLogger) Info(arg0 string, arg1 map[string]string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Info", arg0, arg1)
+}
+
+// Info indicates an expected call of Info
+func (mr *MockLoggerMockRecorder) Info(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockLogger)(nil).Info), arg0, arg1)
+}
+
+// Error mocks base method
+func (m *MockLogger) Error(arg0 string, arg1 map[string]string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Error", arg0, arg1)
+}
+
+// Error indicates an expected call of Error
+func (mr *MockLoggerMockRecorder) Error(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Error", reflect.TypeOf((*MockLogger)(nil).Error), arg0, arg1)
+}
+
+// Debug mocks base method
+func (m *MockLogger) Debug(arg0 string, arg1 map[string]string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Debug", arg0, arg1)
+}
+
+// Debug indicates an expected call of Debug
+func (mr *MockLoggerMockRecorder) Debug(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Debug", reflect.TypeOf((*MockLogger)(nil).Debug), arg0, arg1)
+}
+
+// Event mocks base method
+func (m *MockLogger) Event() string {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Event")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+// Event indicates an expected call of Event
+func (mr *MockLoggerMockRecorder) Event() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Event", reflect.TypeOf((*MockLogger)(nil).Event))
+}
+
+// CreateTime mocks base method
+func (m *MockLogger) CreateTime() time.Time {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateTime")
+	ret0, _ := ret[0].(time.Time)
+	return ret0
+}
+
+// CreateTime indicates an expected call of CreateTime
+func (mr *MockLoggerMockRecorder) CreateTime() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTime", reflect.TypeOf((*MockLogger)(nil).CreateTime))
+}
+
+// GetChan mocks base method
+func (m *MockLogger) GetChan() chan []byte {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetChan")
+	ret0, _ := ret[0].(chan []byte)
+	return ret0
+}
+
+// GetChan indicates an expected call of GetChan
+func (mr *MockLoggerMockRecorder) GetChan() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChan", reflect.TypeOf((*MockLogger)(nil).GetChan))
+}
+
+// SetChan mocks base method
+func (m *MockLogger) SetChan(arg0 chan []byte) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetChan", arg0)
+}
+
+// SetChan indicates an expected call of SetChan
+func (mr *MockLoggerMockRecorder) SetChan(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetChan", reflect.TypeOf((*MockLogger)(nil).SetChan), arg0)
+}
+
+// GetWriter mocks base method
+func (m *MockLogger) GetWriter(step, level string) LoggerWriter {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetWriter", step, level)
+	ret0, _ := ret[0].(LoggerWriter)
+	return ret0
+}
+
+// GetWriter indicates an expected call of GetWriter
+func (mr *MockLoggerMockRecorder) GetWriter(step, level interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWriter", reflect.TypeOf((*MockLogger)(nil).GetWriter), step, level)
+}
+
+// MockLoggerWriter is a mock of LoggerWriter interface
+type MockLoggerWriter struct {
+	ctrl     *gomock.Controller
+	recorder *MockLoggerWriterMockRecorder
+}
+
+// MockLoggerWriterMockRecorder is the mock recorder for MockLoggerWriter
+type MockLoggerWriterMockRecorder struct {
+	mock *MockLoggerWriter
+}
+
+// NewMockLoggerWriter creates a new mock instance
+func NewMockLoggerWriter(ctrl *gomock.Controller) *MockLoggerWriter {
+	mock := &MockLoggerWriter{ctrl: ctrl}
+	mock.recorder = &MockLoggerWriterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockLoggerWriter) EXPECT() *MockLoggerWriterMockRecorder {
+	return m.recorder
+}
+
+// Write mocks base method
+func (m *MockLoggerWriter) Write(p []byte) (int, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Write", p)
+	ret0, _ := ret[0].(int)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Write indicates an expected call of Write
+func (mr *MockLoggerWriterMockRecorder) Write(p interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockLoggerWriter)(nil).Write), p)
+}
+
+// SetFormat mocks base method
+func (m *MockLoggerWriter) SetFormat(arg0 string) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "SetFormat", arg0)
+}
+
+// SetFormat indicates an expected call of SetFormat
+func (mr *MockLoggerWriterMockRecorder) SetFormat(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFormat", reflect.TypeOf((*MockLoggerWriter)(nil).SetFormat), arg0)
+}

mockgen.sh

@@ -1,2 +1,5 @@
 #!/bin/bash
 mockgen -source=worker/appm/store/store.go -destination=worker/appm/store/mock_store.go -package=store
+mockgen -source=db/db.go -destination=db/db_mock.go -package=db
+mockgen -source=db/dao/dao.go -destination=db/dao/dao_mock.go -package=dao
+mockgen -source=event/manager.go -destination=event/manager_mock.go -package=event

worker/appm/store/store.go

@@ -79,6 +79,17 @@ const (
 	DeleteEvent EventType = "DELETE"
 )
 
+// PodEventType -
+type PodEventType string
+
+// String -
+func (p PodEventType) String() string {
+	return string(p)
+}
+
+// PodEventTypeOOMKilled -
+var PodEventTypeOOMKilled PodEventType = "OOMKilled"
+
 // Event holds the context of an event.
 type Event struct {
 	Type EventType
@@ -1003,7 +1014,7 @@ func (a *appRuntimeStore) podEventHandler() cache.ResourceEventHandler {
 		UpdateFunc: func(old, new interface{}) {
 			opod := old.(*corev1.Pod)
 			npod := new.(*corev1.Pod)
-			tenantID, serviceID, version, creatorID := parseLabels(npod.GetLabels())
+			_, serviceID, version, creatorID := parseLabels(npod.GetLabels())
 			if serviceID != "" && version != "" && creatorID != "" {
 				appservice, err := a.getAppService(serviceID, version, creatorID, true)
 				if err == conversion.ErrServiceNotFound {
@@ -1011,17 +1022,8 @@ func (a *appRuntimeStore) podEventHandler() cache.ResourceEventHandler {
 				}
 				if appservice != nil {
 					appservice.SetPods(npod)
-					a.analyzePodStatus(npod)
-					oldPodStatus, newPodStatus := &pb.PodStatus{}, &pb.PodStatus{}
-					wutil.DescribePodStatus(opod, oldPodStatus)
-					wutil.DescribePodStatus(npod, newPodStatus)
-					if checkActionFinish(serviceID, "upgrade", "stop", "start", "build") && oldPodStatus.Type != newPodStatus.Type {
-						eventID := createSystemEvent(tenantID, "instance changed", "instance changed; error creating event: %v")
-						logger := event.GetManager().GetLogger(eventID)
-						defer event.GetManager().ReleaseLogger(logger)
-						logrus.Debugf(fmt.Sprintf("instance changed; old instance: %s; new instance: %s", opod.GetName(), npod.GetName()))
-						logger.Info(fmt.Sprintf("instance changed; old instance: %s; new instance: %s", opod.GetName(), npod.GetName()), nil)
-						logger.Info(fmt.Sprintf("instance changed; old status: %s; new status: %s", oldPodStatus.Type.String(), newPodStatus.Type.String()), event.GetLastLoggerOption())
+					if checkActionFinish(serviceID, "upgrade", "stop", "start", "build") {
+						recordUpdateEvent(opod, npod)
 					}
 					return
 				}
@@ -1029,14 +1031,7 @@ func (a *appRuntimeStore) podEventHandler() cache.ResourceEventHandler {
 		},
 		DeleteFunc: func(obj interface{}) {
 			pod := obj.(*corev1.Pod)
-			tenantID, serviceID, version, creatorID := parseLabels(pod.GetLabels())
-			if checkActionFinish(serviceID, "stop") {
-				eventID := createSystemEvent(tenantID, "instance deleted", "instance deleted; error creating event: %v")
-				logger := event.GetManager().GetLogger(eventID)
-				defer event.GetManager().ReleaseLogger(logger)
-				logrus.Debugf(fmt.Sprintf("instance deleted %s", pod.GetName()))
-				logger.Info(fmt.Sprintf("instance deleted %s", pod.GetName()), event.GetLastLoggerOption())
-			}
+			_, serviceID, version, creatorID := parseLabels(pod.GetLabels())
 			if serviceID != "" && version != "" && creatorID != "" {
 				appservice, _ := a.getAppService(serviceID, version, creatorID, false)
 				if appservice != nil {
@@ -1051,14 +1046,16 @@ func (a *appRuntimeStore) podEventHandler() cache.ResourceEventHandler {
 	}
 }
 
-func createSystemEvent(tenantID, optType, msgFormat string) string {
+func createSystemEvent(tenantID, targetID, msgFormat string, optType PodEventType) string {
 	eventID := util.NewUUID()
 	et := &model.ServiceEvent{
 		EventID:   eventID,
 		TenantID:  tenantID,
+		Target:    model.TargetTypeService,
+		TargetID:  targetID,
 		UserName:  "system",
 		StartTime: time.Now().Format(time.RFC3339),
-		OptType:   optType,
+		OptType:   optType.String(),
 	}
 	if err := db.GetManager().ServiceEventDao().AddModel(et); err != nil {
 		logrus.Warningf(msgFormat, err)
@@ -1069,7 +1066,7 @@ func createSystemEvent(tenantID, optType, msgFormat string) string {
 
 func checkActionFinish(serviceID string, optTypes ...string) bool {
 	// TODO: use new opt_type
-	evt, err := db.GetManager().ServiceEventDao().GetBySIDAndType(serviceID, optTypes...)
+	evt, err := db.GetManager().ServiceEventDao().GetByTargetIDAndType(serviceID, optTypes...)
 	if err != nil {
 		if err == gorm.ErrRecordNotFound {
 			return true
@@ -1087,3 +1084,65 @@ func checkActionFinish(serviceID string, optTypes ...string) bool {
 func parseLabels(labels map[string]string) (string, string, string, string) {
 	return labels["tenant_id"], labels["service_id"], labels["version"], labels["creater_id"]
 }
+
+func recordUpdateEvent(old, new *corev1.Pod) {
+	// judge the state of the event
+	oldStatus, newStatus := &pb.PodStatus{}, &pb.PodStatus{}
+	wutil.DescribePodStatus(old, oldStatus)
+	wutil.DescribePodStatus(new, newStatus)
+	if oldStatus.Type == newStatus.Type {
+		return
+	}
+
+	tenantID, serviceID, _, _ := parseLabels(new.GetLabels())
+	// the pod in the pending status has no start time and container statuses
+	for _, cs := range new.Status.ContainerStatuses {
+		state := cs.State
+		if state.Terminated != nil {
+			if state.Terminated.Reason != PodEventTypeOOMKilled.String() {
+				continue
+			}
+			// get last 'OOMKilled' event
+			evt, err := db.GetManager().ServiceEventDao().GetByTargetIDTypeUser(serviceID, PodEventTypeOOMKilled.String(), model.UsernameSystem)
+			if err != nil && err != gorm.ErrRecordNotFound {
+				logrus.Warningf("record update event; error getting event: %v", err)
+				continue
+			}
+			eventID := ""
+			msg := fmt.Sprintf("Instance changed from %s to %s; Reason: %s.", oldStatus.Type.String(), newStatus.Type.String(), state.Terminated.Reason)
+			if err == gorm.ErrRecordNotFound || evt.FinalStatus == model.EventFinalStatusComplete.String() {
+				// create new event
+				eventID = createSystemEvent(tenantID, serviceID, msg, PodEventTypeOOMKilled)
+			} else {
+				eventID = evt.EventID
+			}
+
+			logger := event.GetManager().GetLogger(eventID)
+			defer event.GetManager().ReleaseLogger(logger)
+			logrus.Debugf("Service id: %s; %s.", serviceID, msg)
+			logger.Error(msg, event.GetLoggerOption("failure"))
+		}
+		if state.Running != nil {
+			evt, err := db.GetManager().ServiceEventDao().GetByTargetIDTypeUser(serviceID, PodEventTypeOOMKilled.String(), model.UsernameSystem)
+			if err != nil && err != gorm.ErrRecordNotFound {
+				logrus.Warningf("record update event; error getting event: %v", err)
+				continue
+			}
+			if err == gorm.ErrRecordNotFound || evt.FinalStatus == model.EventFinalStatusComplete.String() {
+				continue
+			}
+			opt := map[string]string{}
+			if time.Now().Sub(state.Running.StartedAt.Time) > 5*time.Minute {
+				opt = event.GetLastLoggerOption()
+			} else {
+				opt = event.GetLoggerOption("failure")
+			}
+			msg := fmt.Sprintf("Instance changed from %s to %s; Started at: %s.", oldStatus.Type.String(), newStatus.Type.String(), state.Running.StartedAt)
+			logger := event.GetManager().GetLogger(evt.EventID)
+			defer event.GetManager().ReleaseLogger(logger)
+			logrus.Debugf("Service id: %s; %s.", serviceID, msg)
+			logger.Error(msg, opt)
+		}
+
+	}
+}

worker/appm/store/store_test.go

@@ -19,13 +19,22 @@
 package store
 
 import (
+	"encoding/json"
+	"io/ioutil"
+	"testing"
+
 	"github.com/eapache/channels"
+	"github.com/golang/mock/gomock"
 	"github.com/goodrain/rainbond/cmd/worker/option"
 	"github.com/goodrain/rainbond/db"
 	"github.com/goodrain/rainbond/db/config"
+	"github.com/goodrain/rainbond/db/dao"
+	"github.com/goodrain/rainbond/db/model"
+	"github.com/goodrain/rainbond/event"
+	"github.com/jinzhu/gorm"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"testing"
 )
 
 func TestAppRuntimeStore_GetTenantResource(t *testing.T) {
@@ -68,6 +77,155 @@ func TestAppRuntimeStore_GetTenantResource(t *testing.T) {
 
 	tenantID := "d22797956503441abce65e40705aac29"
 	resource := store.GetTenantResource(tenantID)
-	t.Logf("%+v",resource)
-	//t.Error("")
+	t.Logf("%+v", resource)
+}
+
+func podFromJSONFile(t *testing.T, filename string) *corev1.Pod {
+	jsonfile, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatalf("failed to read file '%s': %v", filename, err)
+	}
+
+	var pod corev1.Pod
+	if err := json.Unmarshal(jsonfile, &pod); err != nil {
+		t.Fatalf("file: %s; failed to unmarshalling json: %v", filename, err)
+	}
+
+	return &pod
+}
+
+func TestRecordUpdateEvent(t *testing.T) {
+	tests := []struct {
+		name, oldPodFile, newPodFile                   string
+		eventID, tenantID, targetID, optType, username string
+		finalStatus                                    model.EventFinalStatus
+		eventErr                                       error
+		explevel, expstatus                            string
+	}{
+		{
+			name:        "OOMkilled",
+			oldPodFile:  "testdata/pod-pending.json",
+			newPodFile:  "testdata/pod-oom-killed.json",
+			eventID:     "event id",
+			tenantID:    "6e22adb70c114b1d9a46d17d8146ba37",
+			targetID:    "135c3e10e3be34337bde752449a07e4c",
+			optType:     "OOMKilled",
+			username:    model.UsernameSystem,
+			finalStatus: model.EventFinalStatusRunning,
+			eventErr:    gorm.ErrRecordNotFound,
+			explevel:    "error",
+			expstatus:   "failure",
+		},
+		{
+			name:        "duplicated OOMkilled",
+			oldPodFile:  "testdata/pod-pending.json",
+			newPodFile:  "testdata/pod-oom-killed.json",
+			eventID:     "event id",
+			tenantID:    "6e22adb70c114b1d9a46d17d8146ba37",
+			targetID:    "135c3e10e3be34337bde752449a07e4c",
+			optType:     "OOMKilled",
+			username:    model.UsernameSystem,
+			finalStatus: model.EventFinalStatusRunning,
+			eventErr:    nil,
+			explevel:    "error",
+			expstatus:   "failure",
+		},
+		{
+			name:       "running temporarily",
+			oldPodFile: "testdata/pod-oom-killed.json",
+			newPodFile: "testdata/pod-running-temporarily.json",
+			eventID:    "event id",
+			tenantID:   "6e22adb70c114b1d9a46d17d8146ba37",
+			targetID:   "135c3e10e3be34337bde752449a07e4c",
+			optType:    "OOMKilled",
+			username:   model.UsernameSystem,
+			eventErr:   nil,
+			explevel:   "error",
+			expstatus:  "failure",
+		},
+		{
+			name:       "running",
+			oldPodFile: "testdata/pod-oom-killed.json",
+			newPodFile: "testdata/pod-running.json",
+			eventID:    "event id",
+			tenantID:   "6e22adb70c114b1d9a46d17d8146ba37",
+			targetID:   "135c3e10e3be34337bde752449a07e4c",
+			optType:    "OOMKilled",
+			username:   model.UsernameSystem,
+			eventErr:   nil,
+			explevel:   "error",
+			expstatus:  "success",
+		},
+	}
+
+	for idx := range tests {
+		tc := tests[idx]
+		t.Run(tc.name, func(t *testing.T) {
+			stopCh := make(chan struct{})
+
+			old := podFromJSONFile(t, tc.oldPodFile)
+			new := podFromJSONFile(t, tc.newPodFile)
+
+			ctrl := gomock.NewController(t)
+			defer ctrl.Finish()
+
+			// mock db
+			dbmanager := db.NewMockManager(ctrl)
+			db.SetManager(dbmanager)
+			serviceEventDao := dao.NewMockEventDao(ctrl)
+			dbmanager.EXPECT().ServiceEventDao().AnyTimes().Return(serviceEventDao)
+			var evt *model.ServiceEvent
+			if tc.eventErr == nil {
+				evt = &model.ServiceEvent{
+					EventID:     tc.eventID,
+					TenantID:    tc.tenantID,
+					TargetID:    tc.targetID,
+					UserName:    tc.username,
+					OptType:     tc.optType,
+					FinalStatus: model.EventFinalStatusRunning.String(),
+				}
+				evt.CreatedAt = new.Status.StartTime.Time
+			}
+			serviceEventDao.EXPECT().AddModel(gomock.Any()).AnyTimes().Return(nil)
+			serviceEventDao.EXPECT().GetByTargetIDTypeUser(tc.targetID, tc.optType, tc.username).Return(evt, tc.eventErr)
+
+			// mock event manager
+			lm := event.NewMockManager(ctrl)
+			event.SetManager(lm)
+			sendCh := make(chan []byte)
+			l := event.NewLogger(tc.eventID, sendCh)
+			lm.EXPECT().GetLogger(gomock.Any()).Return(l).AnyTimes()
+			lm.EXPECT().ReleaseLogger(l)
+
+			// receive message from logger
+			go func(sendCh chan []byte) {
+				for {
+					select {
+					case msg := <-sendCh:
+						var data map[string]string
+						if err := json.Unmarshal(msg, &data); err != nil {
+							t.Logf("Recevied message: %s", string(msg))
+						}
+						level := data["level"]
+						status := data["status"]
+						if level == "" || status == "" {
+							t.Errorf("Recevied wrong message: %s; expected field 'level' and 'status'", string(msg))
+						} else {
+							if level != tc.explevel {
+								t.Errorf("Expected %s for level, but returned %s", tc.explevel, level)
+							}
+							if status != tc.expstatus {
+								t.Errorf("Expected %s for status, but returned %s\n", tc.expstatus, status)
+							}
+						}
+
+						close(stopCh)
+					}
+				}
+			}(sendCh)
+
+			recordUpdateEvent(old, new)
+			<-stopCh
+		})
+	}
 }

worker/appm/store/testdata/pod-oom-killed.json

@@ -0,0 +1,250 @@
+{
+    "apiVersion": "v1",
+    "kind": "Pod",
+    "metadata": {
+        "name": "135c3e10e3be34337bde752449a07e4c-deployment-7bbbb85dc5-mmwzk",
+        "generateName": "135c3e10e3be34337bde752449a07e4c-deployment-7bbbb85dc5-",
+        "namespace": "6e22adb70c114b1d9a46d17d8146ba37",
+        "selfLink": "/api/v1/namespaces/6e22adb70c114b1d9a46d17d8146ba37/pods/135c3e10e3be34337bde752449a07e4c-deployment-7bbbb85dc5-mmwzk",
+        "uid": "e59b60c1-c64f-11e9-a0de-468210d133b9",
+        "resourceVersion": "1335700",
+        "creationTimestamp": "2019-08-24T09:16:46Z",
+        "labels": {
+            "creater": "Rainbond",
+            "creater_id": "1566638206772576378",
+            "name": "gra07e4c",
+            "pod-template-hash": "3666641871",
+            "service_alias": "gra07e4c",
+            "service_id": "135c3e10e3be34337bde752449a07e4c",
+            "tenant_id": "6e22adb70c114b1d9a46d17d8146ba37",
+            "tenant_name": "03l9piwh",
+            "version": "20190824165943"
+        },
+        "annotations": {
+            "rainbond.com/tolerate-unready-endpoints": "true"
+        },
+        "ownerReferences": [
+            {
+                "apiVersion": "extensions/v1beta1",
+                "kind": "ReplicaSet",
+                "name": "135c3e10e3be34337bde752449a07e4c-deployment-7bbbb85dc5",
+                "uid": "e596630a-c64f-11e9-a0de-468210d133b9",
+                "controller": true,
+                "blockOwnerDeletion": true
+            }
+        ]
+    },
+    "spec": {
+        "volumes": [
+            {
+                "name": "default-token-8bdg9",
+                "secret": {
+                    "secretName": "default-token-8bdg9",
+                    "defaultMode": 420
+                }
+            }
+        ],
+        "containers": [
+            {
+                "name": "135c3e10e3be34337bde752449a07e4c",
+                "image": "goodrain.me/135c3e10e3be34337bde752449a07e4c:20190824165943",
+                "ports": [
+                    {
+                        "containerPort": 5000,
+                        "protocol": "TCP"
+                    }
+                ],
+                "env": [
+                    {
+                        "name": "LOGGER_DRIVER_NAME",
+                        "value": "streamlog"
+                    },
+                    {
+                        "name": "PORT",
+                        "value": "5000"
+                    },
+                    {
+                        "name": "PROTOCOL",
+                        "value": "http"
+                    },
+                    {
+                        "name": "DOMAIN_5000",
+                        "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+                    },
+                    {
+                        "name": "DOMAIN_PROTOCOL_5000",
+                        "value": "http"
+                    },
+                    {
+                        "name": "DOMAIN",
+                        "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+                    },
+                    {
+                        "name": "DOMAIN_PROTOCOL",
+                        "value": "http"
+                    },
+                    {
+                        "name": "MONITOR_PORT",
+                        "value": "5000"
+                    },
+                    {
+                        "name": "CUR_NET",
+                        "value": "midonet"
+                    },
+                    {
+                        "name": "TENANT_ID",
+                        "value": "6e22adb70c114b1d9a46d17d8146ba37"
+                    },
+                    {
+                        "name": "SERVICE_ID",
+                        "value": "135c3e10e3be34337bde752449a07e4c"
+                    },
+                    {
+                        "name": "MEMORY_SIZE",
+                        "value": "small"
+                    },
+                    {
+                        "name": "SERVICE_NAME",
+                        "value": "gra07e4c"
+                    },
+                    {
+                        "name": "SERVICE_EXTEND_METHOD",
+                        "value": "stateless"
+                    },
+                    {
+                        "name": "SERVICE_POD_NUM",
+                        "value": "1"
+                    },
+                    {
+                        "name": "HOST_IP",
+                        "valueFrom": {
+                            "fieldRef": {
+                                "apiVersion": "v1",
+                                "fieldPath": "status.hostIP"
+                            }
+                        }
+                    },
+                    {
+                        "name": "POD_IP",
+                        "valueFrom": {
+                            "fieldRef": {
+                                "apiVersion": "v1",
+                                "fieldPath": "status.podIP"
+                            }
+                        }
+                    }
+                ],
+                "resources": {
+                    "limits": {
+                        "cpu": "0",
+                        "memory": "64Mi"
+                    },
+                    "requests": {
+                        "cpu": "0",
+                        "memory": "64Mi"
+                    }
+                },
+                "volumeMounts": [
+                    {
+                        "name": "default-token-8bdg9",
+                        "readOnly": true,
+                        "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+                    }
+                ],
+                "terminationMessagePath": "/dev/termination-log",
+                "terminationMessagePolicy": "File",
+                "imagePullPolicy": "IfNotPresent"
+            }
+        ],
+        "restartPolicy": "Always",
+        "terminationGracePeriodSeconds": 30,
+        "dnsPolicy": "ClusterFirst",
+        "serviceAccountName": "default",
+        "serviceAccount": "default",
+        "nodeName": "c6996a36-d345-4d11-9bc2-a772706308e5",
+        "securityContext": {},
+        "affinity": {
+            "nodeAffinity": {
+                "requiredDuringSchedulingIgnoredDuringExecution": {
+                    "nodeSelectorTerms": [
+                        {
+                            "matchExpressions": [
+                                {
+                                    "key": "beta.kubernetes.io/os",
+                                    "operator": "NotIn",
+                                    "values": [
+                                        "windows"
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                }
+            }
+        },
+        "schedulerName": "default-scheduler",
+        "tolerations": [
+            {
+                "key": "node.kubernetes.io/not-ready",
+                "operator": "Exists",
+                "effect": "NoExecute",
+                "tolerationSeconds": 300
+            },
+            {
+                "key": "node.kubernetes.io/unreachable",
+                "operator": "Exists",
+                "effect": "NoExecute",
+                "tolerationSeconds": 300
+            }
+        ]
+    },
+    "status": {
+        "phase": "Running",
+        "conditions": [
+            {
+                "type": "Initialized",
+                "status": "True",
+                "lastProbeTime": null,
+                "lastTransitionTime": "2019-08-24T09:16:46Z"
+            },
+            {
+                "type": "Ready",
+                "status": "False",
+                "lastProbeTime": null,
+                "lastTransitionTime": "2019-08-24T09:32:11Z",
+                "reason": "ContainersNotReady",
+                "message": "containers with unready status: [135c3e10e3be34337bde752449a07e4c]"
+            },
+            {
+                "type": "PodScheduled",
+                "status": "True",
+                "lastProbeTime": null,
+                "lastTransitionTime": "2019-08-24T09:16:46Z"
+            }
+        ],
+        "hostIP": "192.168.2.172",
+        "podIP": "10.0.237.8",
+        "startTime": "2019-08-24T09:16:46Z",
+        "containerStatuses": [
+            {
+                "name": "135c3e10e3be34337bde752449a07e4c",
+                "state": {
+                    "terminated": {
+                        "exitCode": 137,
+                        "reason": "OOMKilled",
+                        "startedAt": "2019-08-24T09:31:06Z",
+                        "finishedAt": "2019-08-24T09:32:11Z",
+                        "containerID": "docker://71cde7288e0ae38f4e800ce1e19e32f85480ae3c10df7d4972084fa583b73728"
+                    }
+                },
+                "lastState": {},
+                "ready": false,
+                "restartCount": 0,
+                "image": "goodrain.me/135c3e10e3be34337bde752449a07e4c:20190824165943",
+                "imageID": "docker-pullable://goodrain.me/135c3e10e3be34337bde752449a07e4c@sha256:034e4838411bc72656e30ad3719fddbf86129bc63e7f488547d36b9a1f5c4084",
+                "containerID": "docker://71cde7288e0ae38f4e800ce1e19e32f85480ae3c10df7d4972084fa583b73728"
+            }
+        ],
+        "qosClass": "Burstable"
+    }
+}

worker/appm/store/testdata/pod-pending.json

@@ -0,0 +1,211 @@
+{
+    "metadata": {
+      "name": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-m7j8x",
+      "generateName": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-",
+      "namespace": "6e22adb70c114b1d9a46d17d8146ba37",
+      "selfLink": "/api/v1/namespaces/6e22adb70c114b1d9a46d17d8146ba37/pods/135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-m7j8x",
+      "uid": "ca2077d0-c654-11e9-a0de-468210d133b9",
+      "resourceVersion": "1340493",
+      "creationTimestamp": "2019-08-24T09:51:48Z",
+      "labels": {
+        "creater": "Rainbond",
+        "creater_id": "1566640308184453616",
+        "name": "gra07e4c",
+        "pod-template-hash": "3468213781",
+        "service_alias": "gra07e4c",
+        "service_id": "135c3e10e3be34337bde752449a07e4c",
+        "tenant_id": "6e22adb70c114b1d9a46d17d8146ba37",
+        "tenant_name": "03l9piwh",
+        "version": "20190824165943"
+      },
+      "annotations": {
+        "rainbond.com/tolerate-unready-endpoints": "true"
+      },
+      "ownerReferences": [
+        {
+          "apiVersion": "extensions/v1beta1",
+          "kind": "ReplicaSet",
+          "name": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5",
+          "uid": "ca1ed265-c654-11e9-a0de-468210d133b9",
+          "controller": true,
+          "blockOwnerDeletion": true
+        }
+      ]
+    },
+    "spec": {
+      "volumes": [
+        {
+          "name": "default-token-8bdg9",
+          "secret": {
+            "secretName": "default-token-8bdg9",
+            "defaultMode": 420
+          }
+        }
+      ],
+      "containers": [
+        {
+          "name": "135c3e10e3be34337bde752449a07e4c",
+          "image": "goodrain.me/135c3e10e3be34337bde752449a07e4c:20190824165943",
+          "ports": [
+            {
+              "containerPort": 5000,
+              "protocol": "TCP"
+            }
+          ],
+          "env": [
+            {
+              "name": "LOGGER_DRIVER_NAME",
+              "value": "streamlog"
+            },
+            {
+              "name": "PORT",
+              "value": "5000"
+            },
+            {
+              "name": "PROTOCOL",
+              "value": "http"
+            },
+            {
+              "name": "DOMAIN_5000",
+              "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+            },
+            {
+              "name": "DOMAIN_PROTOCOL_5000",
+              "value": "http"
+            },
+            {
+              "name": "DOMAIN",
+              "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+            },
+            {
+              "name": "DOMAIN_PROTOCOL",
+              "value": "http"
+            },
+            {
+              "name": "MONITOR_PORT",
+              "value": "5000"
+            },
+            {
+              "name": "CUR_NET",
+              "value": "midonet"
+            },
+            {
+              "name": "TENANT_ID",
+              "value": "6e22adb70c114b1d9a46d17d8146ba37"
+            },
+            {
+              "name": "SERVICE_ID",
+              "value": "135c3e10e3be34337bde752449a07e4c"
+            },
+            {
+              "name": "MEMORY_SIZE",
+              "value": "small"
+            },
+            {
+              "name": "SERVICE_NAME",
+              "value": "gra07e4c"
+            },
+            {
+              "name": "SERVICE_EXTEND_METHOD",
+              "value": "stateless"
+            },
+            {
+              "name": "SERVICE_POD_NUM",
+              "value": "1"
+            },
+            {
+              "name": "HOST_IP",
+              "valueFrom": {
+                "fieldRef": {
+                  "apiVersion": "v1",
+                  "fieldPath": "status.hostIP"
+                }
+              }
+            },
+            {
+              "name": "POD_IP",
+              "valueFrom": {
+                "fieldRef": {
+                  "apiVersion": "v1",
+                  "fieldPath": "status.podIP"
+                }
+              }
+            }
+          ],
+          "resources": {
+            "limits": {
+              "cpu": "0",
+              "memory": "64Mi"
+            },
+            "requests": {
+              "cpu": "0",
+              "memory": "64Mi"
+            }
+          },
+          "volumeMounts": [
+            {
+              "name": "default-token-8bdg9",
+              "readOnly": true,
+              "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+            }
+          ],
+          "terminationMessagePath": "/dev/termination-log",
+          "terminationMessagePolicy": "File",
+          "imagePullPolicy": "IfNotPresent"
+        }
+      ],
+      "restartPolicy": "Always",
+      "terminationGracePeriodSeconds": 30,
+      "dnsPolicy": "ClusterFirst",
+      "serviceAccountName": "default",
+      "serviceAccount": "default",
+      "nodeName": "c6996a36-d345-4d11-9bc2-a772706308e5",
+      "securityContext": {},
+      "affinity": {
+        "nodeAffinity": {
+          "requiredDuringSchedulingIgnoredDuringExecution": {
+            "nodeSelectorTerms": [
+              {
+                "matchExpressions": [
+                  {
+                    "key": "beta.kubernetes.io/os",
+                    "operator": "NotIn",
+                    "values": [
+                      "windows"
+                    ]
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      },
+      "schedulerName": "default-scheduler",
+      "tolerations": [
+        {
+          "key": "node.kubernetes.io/not-ready",
+          "operator": "Exists",
+          "effect": "NoExecute",
+          "tolerationSeconds": 300
+        },
+        {
+          "key": "node.kubernetes.io/unreachable",
+          "operator": "Exists",
+          "effect": "NoExecute",
+          "tolerationSeconds": 300
+        }
+      ]
+    },
+    "status": {
+      "phase": "Pending",
+      "conditions": [
+        {
+          "type": "PodScheduled",
+          "status": "True",
+          "lastProbeTime": null,
+          "lastTransitionTime": "2019-08-24T09:51:48Z"
+        }
+      ],
+      "qosClass": "Burstable"
+    }
+  }

worker/appm/store/testdata/pod-running-temporarily.json

@@ -0,0 +1,242 @@
+{
+    "metadata": {
+      "name": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-m7j8x",
+      "generateName": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-",
+      "namespace": "6e22adb70c114b1d9a46d17d8146ba37",
+      "selfLink": "/api/v1/namespaces/6e22adb70c114b1d9a46d17d8146ba37/pods/135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-m7j8x",
+      "uid": "ca2077d0-c654-11e9-a0de-468210d133b9",
+      "resourceVersion": "1340515",
+      "creationTimestamp": "2019-08-24T09:51:48Z",
+      "labels": {
+        "creater": "Rainbond",
+        "creater_id": "1566640308184453616",
+        "name": "gra07e4c",
+        "pod-template-hash": "3468213781",
+        "service_alias": "gra07e4c",
+        "service_id": "135c3e10e3be34337bde752449a07e4c",
+        "tenant_id": "6e22adb70c114b1d9a46d17d8146ba37",
+        "tenant_name": "03l9piwh",
+        "version": "20190824165943"
+      },
+      "annotations": {
+        "rainbond.com/tolerate-unready-endpoints": "true"
+      },
+      "ownerReferences": [
+        {
+          "apiVersion": "extensions/v1beta1",
+          "kind": "ReplicaSet",
+          "name": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5",
+          "uid": "ca1ed265-c654-11e9-a0de-468210d133b9",
+          "controller": true,
+          "blockOwnerDeletion": true
+        }
+      ]
+    },
+    "spec": {
+      "volumes": [
+        {
+          "name": "default-token-8bdg9",
+          "secret": {
+            "secretName": "default-token-8bdg9",
+            "defaultMode": 420
+          }
+        }
+      ],
+      "containers": [
+        {
+          "name": "135c3e10e3be34337bde752449a07e4c",
+          "image": "goodrain.me/135c3e10e3be34337bde752449a07e4c:20190824165943",
+          "ports": [
+            {
+              "containerPort": 5000,
+              "protocol": "TCP"
+            }
+          ],
+          "env": [
+            {
+              "name": "LOGGER_DRIVER_NAME",
+              "value": "streamlog"
+            },
+            {
+              "name": "PORT",
+              "value": "5000"
+            },
+            {
+              "name": "PROTOCOL",
+              "value": "http"
+            },
+            {
+              "name": "DOMAIN_5000",
+              "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+            },
+            {
+              "name": "DOMAIN_PROTOCOL_5000",
+              "value": "http"
+            },
+            {
+              "name": "DOMAIN",
+              "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+            },
+            {
+              "name": "DOMAIN_PROTOCOL",
+              "value": "http"
+            },
+            {
+              "name": "MONITOR_PORT",
+              "value": "5000"
+            },
+            {
+              "name": "CUR_NET",
+              "value": "midonet"
+            },
+            {
+              "name": "TENANT_ID",
+              "value": "6e22adb70c114b1d9a46d17d8146ba37"
+            },
+            {
+              "name": "SERVICE_ID",
+              "value": "135c3e10e3be34337bde752449a07e4c"
+            },
+            {
+              "name": "MEMORY_SIZE",
+              "value": "small"
+            },
+            {
+              "name": "SERVICE_NAME",
+              "value": "gra07e4c"
+            },
+            {
+              "name": "SERVICE_EXTEND_METHOD",
+              "value": "stateless"
+            },
+            {
+              "name": "SERVICE_POD_NUM",
+              "value": "1"
+            },
+            {
+              "name": "HOST_IP",
+              "valueFrom": {
+                "fieldRef": {
+                  "apiVersion": "v1",
+                  "fieldPath": "status.hostIP"
+                }
+              }
+            },
+            {
+              "name": "POD_IP",
+              "valueFrom": {
+                "fieldRef": {
+                  "apiVersion": "v1",
+                  "fieldPath": "status.podIP"
+                }
+              }
+            }
+          ],
+          "resources": {
+            "limits": {
+              "cpu": "0",
+              "memory": "64Mi"
+            },
+            "requests": {
+              "cpu": "0",
+              "memory": "64Mi"
+            }
+          },
+          "volumeMounts": [
+            {
+              "name": "default-token-8bdg9",
+              "readOnly": true,
+              "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+            }
+          ],
+          "terminationMessagePath": "/dev/termination-log",
+          "terminationMessagePolicy": "File",
+          "imagePullPolicy": "IfNotPresent"
+        }
+      ],
+      "restartPolicy": "Always",
+      "terminationGracePeriodSeconds": 30,
+      "dnsPolicy": "ClusterFirst",
+      "serviceAccountName": "default",
+      "serviceAccount": "default",
+      "nodeName": "c6996a36-d345-4d11-9bc2-a772706308e5",
+      "securityContext": {},
+      "affinity": {
+        "nodeAffinity": {
+          "requiredDuringSchedulingIgnoredDuringExecution": {
+            "nodeSelectorTerms": [
+              {
+                "matchExpressions": [
+                  {
+                    "key": "beta.kubernetes.io/os",
+                    "operator": "NotIn",
+                    "values": [
+                      "windows"
+                    ]
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      },
+      "schedulerName": "default-scheduler",
+      "tolerations": [
+        {
+          "key": "node.kubernetes.io/not-ready",
+          "operator": "Exists",
+          "effect": "NoExecute",
+          "tolerationSeconds": 300
+        },
+        {
+          "key": "node.kubernetes.io/unreachable",
+          "operator": "Exists",
+          "effect": "NoExecute",
+          "tolerationSeconds": 300
+        }
+      ]
+    },
+    "status": {
+      "phase": "Running",
+      "conditions": [
+        {
+          "type": "Initialized",
+          "status": "True",
+          "lastProbeTime": null,
+          "lastTransitionTime": "2019-08-24T09:51:48Z"
+        },
+        {
+          "type": "Ready",
+          "status": "True",
+          "lastProbeTime": null,
+          "lastTransitionTime": "2019-08-24T09:51:49Z"
+        },
+        {
+          "type": "PodScheduled",
+          "status": "True",
+          "lastProbeTime": null,
+          "lastTransitionTime": "2019-08-24T09:51:48Z"
+        }
+      ],
+      "hostIP": "192.168.2.172",
+      "podIP": "10.0.237.9",
+      "startTime": "2019-08-24T09:51:48Z",
+      "containerStatuses": [
+        {
+          "name": "135c3e10e3be34337bde752449a07e4c",
+          "state": {
+            "running": {
+              "startedAt": "2019-08-24T09:51:49Z"
+            }
+          },
+          "lastState": {},
+          "ready": true,
+          "restartCount": 0,
+          "image": "goodrain.me/135c3e10e3be34337bde752449a07e4c:20190824165943",
+          "imageID": "docker-pullable://goodrain.me/135c3e10e3be34337bde752449a07e4c@sha256:034e4838411bc72656e30ad3719fddbf86129bc63e7f488547d36b9a1f5c4084",
+          "containerID": "docker://7a7efa359db9449a9271d61c18edee3ed773b5b57fed29a937bd5f247cc52ec0"
+        }
+      ],
+      "qosClass": "Burstable"
+    }
+  }

worker/appm/store/testdata/pod-running.json

@@ -0,0 +1,242 @@
+{
+    "metadata": {
+      "name": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-m7j8x",
+      "generateName": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-",
+      "namespace": "6e22adb70c114b1d9a46d17d8146ba37",
+      "selfLink": "/api/v1/namespaces/6e22adb70c114b1d9a46d17d8146ba37/pods/135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5-m7j8x",
+      "uid": "ca2077d0-c654-11e9-a0de-468210d133b9",
+      "resourceVersion": "1340515",
+      "creationTimestamp": "2019-08-24T09:51:48Z",
+      "labels": {
+        "creater": "Rainbond",
+        "creater_id": "1566640308184453616",
+        "name": "gra07e4c",
+        "pod-template-hash": "3468213781",
+        "service_alias": "gra07e4c",
+        "service_id": "135c3e10e3be34337bde752449a07e4c",
+        "tenant_id": "6e22adb70c114b1d9a46d17d8146ba37",
+        "tenant_name": "03l9piwh",
+        "version": "20190824165943"
+      },
+      "annotations": {
+        "rainbond.com/tolerate-unready-endpoints": "true"
+      },
+      "ownerReferences": [
+        {
+          "apiVersion": "extensions/v1beta1",
+          "kind": "ReplicaSet",
+          "name": "135c3e10e3be34337bde752449a07e4c-deployment-78bd657cd5",
+          "uid": "ca1ed265-c654-11e9-a0de-468210d133b9",
+          "controller": true,
+          "blockOwnerDeletion": true
+        }
+      ]
+    },
+    "spec": {
+      "volumes": [
+        {
+          "name": "default-token-8bdg9",
+          "secret": {
+            "secretName": "default-token-8bdg9",
+            "defaultMode": 420
+          }
+        }
+      ],
+      "containers": [
+        {
+          "name": "135c3e10e3be34337bde752449a07e4c",
+          "image": "goodrain.me/135c3e10e3be34337bde752449a07e4c:20190824165943",
+          "ports": [
+            {
+              "containerPort": 5000,
+              "protocol": "TCP"
+            }
+          ],
+          "env": [
+            {
+              "name": "LOGGER_DRIVER_NAME",
+              "value": "streamlog"
+            },
+            {
+              "name": "PORT",
+              "value": "5000"
+            },
+            {
+              "name": "PROTOCOL",
+              "value": "http"
+            },
+            {
+              "name": "DOMAIN_5000",
+              "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+            },
+            {
+              "name": "DOMAIN_PROTOCOL_5000",
+              "value": "http"
+            },
+            {
+              "name": "DOMAIN",
+              "value": "5000.gra07e4c.03l9piwh.164de4.grapps.cn"
+            },
+            {
+              "name": "DOMAIN_PROTOCOL",
+              "value": "http"
+            },
+            {
+              "name": "MONITOR_PORT",
+              "value": "5000"
+            },
+            {
+              "name": "CUR_NET",
+              "value": "midonet"
+            },
+            {
+              "name": "TENANT_ID",
+              "value": "6e22adb70c114b1d9a46d17d8146ba37"
+            },
+            {
+              "name": "SERVICE_ID",
+              "value": "135c3e10e3be34337bde752449a07e4c"
+            },
+            {
+              "name": "MEMORY_SIZE",
+              "value": "small"
+            },
+            {
+              "name": "SERVICE_NAME",
+              "value": "gra07e4c"
+            },
+            {
+              "name": "SERVICE_EXTEND_METHOD",
+              "value": "stateless"
+            },
+            {
+              "name": "SERVICE_POD_NUM",
+              "value": "1"
+            },
+            {
+              "name": "HOST_IP",
+              "valueFrom": {
+                "fieldRef": {
+                  "apiVersion": "v1",
+                  "fieldPath": "status.hostIP"
+                }
+              }
+            },
+            {
+              "name": "POD_IP",
+              "valueFrom": {
+                "fieldRef": {
+                  "apiVersion": "v1",
+                  "fieldPath": "status.podIP"
+                }
+              }
+            }
+          ],
+          "resources": {
+            "limits": {
+              "cpu": "0",
+              "memory": "64Mi"
+            },
+            "requests": {
+              "cpu": "0",
+              "memory": "64Mi"
+            }
+          },
+          "volumeMounts": [
+            {
+              "name": "default-token-8bdg9",
+              "readOnly": true,
+              "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
+            }
+          ],
+          "terminationMessagePath": "/dev/termination-log",
+          "terminationMessagePolicy": "File",
+          "imagePullPolicy": "IfNotPresent"
+        }
+      ],
+      "restartPolicy": "Always",
+      "terminationGracePeriodSeconds": 30,
+      "dnsPolicy": "ClusterFirst",
+      "serviceAccountName": "default",
+      "serviceAccount": "default",
+      "nodeName": "c6996a36-d345-4d11-9bc2-a772706308e5",
+      "securityContext": {},
+      "affinity": {
+        "nodeAffinity": {
+          "requiredDuringSchedulingIgnoredDuringExecution": {
+            "nodeSelectorTerms": [
+              {
+                "matchExpressions": [
+                  {
+                    "key": "beta.kubernetes.io/os",
+                    "operator": "NotIn",
+                    "values": [
+                      "windows"
+                    ]
+                  }
+                ]
+              }
+            ]
+          }
+        }
+      },
+      "schedulerName": "default-scheduler",
+      "tolerations": [
+        {
+          "key": "node.kubernetes.io/not-ready",
+          "operator": "Exists",
+          "effect": "NoExecute",
+          "tolerationSeconds": 300
+        },
+        {
+          "key": "node.kubernetes.io/unreachable",
+          "operator": "Exists",
+          "effect": "NoExecute",
+          "tolerationSeconds": 300
+        }
+      ]
+    },
+    "status": {
+      "phase": "Running",
+      "conditions": [
+        {
+          "type": "Initialized",
+          "status": "True",
+          "lastProbeTime": null,
+          "lastTransitionTime": "2019-08-24T09:51:48Z"
+        },
+        {
+          "type": "Ready",
+          "status": "True",
+          "lastProbeTime": null,
+          "lastTransitionTime": "2019-08-24T09:51:49Z"
+        },
+        {
+          "type": "PodScheduled",
+          "status": "True",
+          "lastProbeTime": null,
+          "lastTransitionTime": "2019-08-24T09:51:48Z"
+        }
+      ],
+      "hostIP": "192.168.2.172",
+      "podIP": "10.0.237.9",
+      "startTime": "2019-08-24T09:51:48Z",
+      "containerStatuses": [
+        {
+          "name": "135c3e10e3be34337bde752449a07e4c",
+          "state": {
+            "running": {
+              "startedAt": "2019-08-24T09:53:49Z"
+            }
+          },
+          "lastState": {},
+          "ready": true,
+          "restartCount": 0,
+          "image": "goodrain.me/135c3e10e3be34337bde752449a07e4c:20190824165943",
+          "imageID": "docker-pullable://goodrain.me/135c3e10e3be34337bde752449a07e4c@sha256:034e4838411bc72656e30ad3719fddbf86129bc63e7f488547d36b9a1f5c4084",
+          "containerID": "docker://7a7efa359db9449a9271d61c18edee3ed773b5b57fed29a937bd5f247cc52ec0"
+        }
+      ],
+      "qosClass": "Burstable"
+    }
+  }