From 843038e65cb389caa15b42d03ec57a35b9a2737e Mon Sep 17 00:00:00 2001 From: GLYASAI Date: Sun, 1 Sep 2019 22:42:32 +0800 Subject: [PATCH] [ADD] pod event type 'EventTypeAbnormalRecovery' --- worker/master/podevent/podevent.go | 47 ++++++++++++++----------- worker/master/podevent/podevent_test.go | 20 +++++------ 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/worker/master/podevent/podevent.go b/worker/master/podevent/podevent.go index b71b64a2f..b77dc2d42 100644 --- a/worker/master/podevent/podevent.go +++ b/worker/master/podevent/podevent.go @@ -19,28 +19,28 @@ import ( "k8s.io/client-go/kubernetes" ) -// PodEventType - -type PodEventType string +// EventType - +type EventType string // String - -func (p PodEventType) String() string { +func (p EventType) String() string { return string(p) } -// PodEventTypeOOMKilled - -var PodEventTypeOOMKilled PodEventType = "OOMKilled" +// EventTypeOOMKilled - +var EventTypeOOMKilled EventType = "OOMKilled" -// PodEventTypeLivenessProbeFailed - -var PodEventTypeLivenessProbeFailed PodEventType = "LivenessProbeFailed" +// EventTypeAbnormalExited container exits abnormally +var EventTypeAbnormalExited EventType = "AbnormalExited" -// PodEventTypeReadinessProbeFailed - -var PodEventTypeReadinessProbeFailed PodEventType = "ReadinessProbeFailed" +// EventTypeLivenessProbeFailed - +var EventTypeLivenessProbeFailed EventType = "LivenessProbeFailed" -// PodEventTypeAbnormalRecovery - -var PodEventTypeAbnormalRecovery PodEventType = "AbnormalRecovery" +// EventTypeReadinessProbeFailed - +var EventTypeReadinessProbeFailed EventType = "ReadinessProbeFailed" -// PodEventTypeAbnormalShtdown - -var PodEventTypeAbnormalShtdown PodEventType = "AbnormalShtdown" +// EventTypeAbnormalRecovery - +var EventTypeAbnormalRecovery EventType = "AbnormalRecovery" // PodEvent - type PodEvent struct { @@ -122,9 +122,9 @@ func recordUpdateEvent(clientset kubernetes.Interface, pod *corev1.Pod, f determ loggerOpt := event.GetLoggerOption("failure") if time.Now().Sub(state.Running.StartedAt.Time) > 2*time.Minute { loggerOpt = event.GetCallbackLoggerOption() - _, err := createSystemEvent(tenantID, serviceID, pod.GetName(), PodEventTypeAbnormalRecovery.String(), model.EventStatusSuccess.String()) + _, err := createSystemEvent(tenantID, serviceID, pod.GetName(), EventTypeAbnormalRecovery.String(), model.EventStatusSuccess.String()) if err != nil { - logrus.Warningf("pod: %s; type: %s; error creating event: %v", pod.GetName(), PodEventTypeAbnormalRecovery.String(), err) + logrus.Warningf("pod: %s; type: %s; error creating event: %v", pod.GetName(), EventTypeAbnormalRecovery.String(), err) continue } } @@ -134,19 +134,24 @@ func recordUpdateEvent(clientset kubernetes.Interface, pod *corev1.Pod, f determ } // determine the type of exception -type determineOptType func(clientset kubernetes.Interface, pod *corev1.Pod, state *corev1.ContainerState, f k8sutil.ListEventsByPod) (PodEventType, string) +type determineOptType func(clientset kubernetes.Interface, pod *corev1.Pod, state *corev1.ContainerState, f k8sutil.ListEventsByPod) (EventType, string) -func defDetermineOptType(clientset kubernetes.Interface, pod *corev1.Pod, state *corev1.ContainerState, f k8sutil.ListEventsByPod) (PodEventType, string) { - if state.Terminated != nil && state.Terminated.Reason == PodEventTypeOOMKilled.String() { - return PodEventTypeOOMKilled, state.Terminated.Reason +func defDetermineOptType(clientset kubernetes.Interface, pod *corev1.Pod, state *corev1.ContainerState, f k8sutil.ListEventsByPod) (EventType, string) { + if state.Terminated != nil { + if state.Terminated.Reason == EventTypeOOMKilled.String() { + return EventTypeOOMKilled, state.Terminated.Reason + } + if state.Terminated.ExitCode != 0 { + return EventTypeAbnormalExited, state.Terminated.Reason + } } events := f(clientset, pod) for _, evt := range events.Items { if strings.Contains(evt.Message, "Liveness probe failed") && state.Waiting != nil { - return PodEventTypeLivenessProbeFailed, evt.Message + return EventTypeLivenessProbeFailed, evt.Message } if strings.Contains(evt.Message, "Readiness probe failed") { - return PodEventTypeReadinessProbeFailed, evt.Message + return EventTypeReadinessProbeFailed, evt.Message } } diff --git a/worker/master/podevent/podevent_test.go b/worker/master/podevent/podevent_test.go index bea349a9c..441bda8e8 100644 --- a/worker/master/podevent/podevent_test.go +++ b/worker/master/podevent/podevent_test.go @@ -53,7 +53,7 @@ func TestRecordUpdateEvent(t *testing.T) { finalStatus model.EventFinalStatus eventErr error explevel, expstatus, message string - optType PodEventType + optType EventType startTime time.Time }{ { @@ -83,7 +83,7 @@ func TestRecordUpdateEvent(t *testing.T) { eventID: "event id", tenantID: "6e22adb70c114b1d9a46d17d8146ba37", serviceID: "135c3e10e3be34337bde752449a07e4c", - optType: PodEventTypeOOMKilled, + optType: EventTypeOOMKilled, eventErr: nil, explevel: "error", expstatus: "failure", @@ -95,7 +95,7 @@ func TestRecordUpdateEvent(t *testing.T) { eventID: "event id", tenantID: "6e22adb70c114b1d9a46d17d8146ba37", serviceID: "135c3e10e3be34337bde752449a07e4c", - optType: PodEventTypeOOMKilled, + optType: EventTypeOOMKilled, eventErr: gorm.ErrRecordNotFound, explevel: "error", expstatus: "failure", @@ -106,7 +106,7 @@ func TestRecordUpdateEvent(t *testing.T) { eventID: "event id", tenantID: "f614a5eddea546c2bbaeb67d381599ee", serviceID: "fa9c83c9198bfee9325804d3b4e7ff06", - optType: PodEventTypeLivenessProbeFailed, + optType: EventTypeLivenessProbeFailed, eventErr: nil, explevel: "error", expstatus: "failure", @@ -117,7 +117,7 @@ func TestRecordUpdateEvent(t *testing.T) { eventID: "event id", tenantID: "f614a5eddea546c2bbaeb67d381599ee", serviceID: "0c3a85977aab7adcc8b3451472d3ee94", - optType: PodEventTypeReadinessProbeFailed, + optType: EventTypeReadinessProbeFailed, eventErr: nil, explevel: "error", expstatus: "failure", @@ -192,7 +192,7 @@ func TestRecordUpdateEvent(t *testing.T) { } }(sendCh) - testDetermineOptType := func(clientset kubernetes.Interface, pod *corev1.Pod, state *corev1.ContainerState, f k8sutil.ListEventsByPod) (PodEventType, string) { + testDetermineOptType := func(clientset kubernetes.Interface, pod *corev1.Pod, state *corev1.ContainerState, f k8sutil.ListEventsByPod) (EventType, string) { return tc.optType, tc.message } @@ -212,11 +212,11 @@ func TestDetermineOptType(t *testing.T) { } tests := []struct { podfile string - podEventType PodEventType + podEventType EventType }{ - {"testdata/pod-readiness.json", PodEventTypeReadinessProbeFailed}, - {"testdata/pod-liveness.json", PodEventTypeLivenessProbeFailed}, - {"testdata/pod-oom-killed.json", PodEventTypeOOMKilled}, + {"testdata/pod-readiness.json", EventTypeReadinessProbeFailed}, + {"testdata/pod-liveness.json", EventTypeLivenessProbeFailed}, + {"testdata/pod-oom-killed.json", EventTypeOOMKilled}, } for idx := range tests { tc := tests[idx]