From 2fde96ff97d99335aef4116f2ee83e0783cbcba5 Mon Sep 17 00:00:00 2001
From: barnett <zengqg@goodrain.com>
Date: Fri, 30 Mar 2018 15:37:36 +0800
Subject: [PATCH] [ADD] add recover in builder

---
 pkg/builder/exector/exector.go       | 37 ++++++++++++++++++++++++++++
 pkg/builder/exector/service_check.go |  8 ++++++
 2 files changed, 45 insertions(+)

diff --git a/pkg/builder/exector/exector.go b/pkg/builder/exector/exector.go
index 19f351a08..56a0042cb 100644
--- a/pkg/builder/exector/exector.go
+++ b/pkg/builder/exector/exector.go
@@ -20,6 +20,7 @@ package exector
 
 import (
 	"fmt"
+	"runtime/debug"
 	"time"
 
 	"github.com/Sirupsen/logrus"
@@ -156,6 +157,12 @@ func (e *exectorManager) buildFromImage(in []byte) {
 	go func() {
 		logrus.Debugf("start build from image worker")
 		defer event.GetManager().ReleaseLogger(i.Logger)
+		defer func() {
+			if r := recover(); r != nil {
+				debug.PrintStack()
+				i.Logger.Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
+			}
+		}()
 		for n := 0; n < 2; n++ {
 			err := i.Run(time.Minute * 30)
 			if err != nil {
@@ -184,6 +191,12 @@ func (e *exectorManager) buildFromSourceCode(in []byte) {
 	go func() {
 		logrus.Debugf("start build from source code")
 		defer event.GetManager().ReleaseLogger(i.Logger)
+		defer func() {
+			if r := recover(); r != nil {
+				debug.PrintStack()
+				i.Logger.Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
+			}
+		}()
 		for n := 0; n < 2; n++ {
 			err := i.Run(time.Minute * 30)
 			if err != nil {
@@ -220,6 +233,12 @@ func (e *exectorManager) buildFromMarketSlug(in []byte) {
 	i.Logger.Info("开始构建应用", map[string]string{"step": "builder-exector", "status": "starting"})
 	go func() {
 		defer event.GetManager().ReleaseLogger(i.Logger)
+		defer func() {
+			if r := recover(); r != nil {
+				debug.PrintStack()
+				i.Logger.Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
+			}
+		}()
 		for n := 0; n < 2; n++ {
 			err := i.Run()
 			if err != nil {
@@ -245,6 +264,12 @@ func (e *exectorManager) appSlug(in []byte) {
 	go func() {
 		logrus.Info("start exec app slug worker")
 		defer event.GetManager().ReleaseLogger(logger)
+		defer func() {
+			if r := recover(); r != nil {
+				debug.PrintStack()
+				logger.Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
+			}
+		}()
 		for i := 0; i < 3; i++ {
 			_, err := w.run(time.Minute * 30)
 			if err != nil {
@@ -413,6 +438,12 @@ func (e *exectorManager) pluginImageBuild1(in []byte) {
 	go func() {
 		logrus.Info("start exec build plugin from image worker")
 		defer event.GetManager().ReleaseLogger(logger)
+		defer func() {
+			if r := recover(); r != nil {
+				debug.PrintStack()
+				logger.Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
+			}
+		}()
 		for i := 0; i < 3; i++ {
 			_, err := w.run(time.Minute * 30)
 			if err != nil {
@@ -438,6 +469,12 @@ func (e *exectorManager) pluginDockerfileBuild1(in []byte) {
 	go func() {
 		logrus.Info("start exec build plugin from image worker")
 		defer event.GetManager().ReleaseLogger(logger)
+		defer func() {
+			if r := recover(); r != nil {
+				debug.PrintStack()
+				logger.Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
+			}
+		}()
 		for i := 0; i < 3; i++ {
 			_, err := w.run(time.Minute * 30)
 			if err != nil {
diff --git a/pkg/builder/exector/service_check.go b/pkg/builder/exector/service_check.go
index b736a7dc5..006723574 100644
--- a/pkg/builder/exector/service_check.go
+++ b/pkg/builder/exector/service_check.go
@@ -21,6 +21,7 @@ package exector
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/ghodss/yaml"
@@ -84,6 +85,13 @@ func (e *exectorManager) serviceCheck(in []byte) {
 		return
 	}
 	logger := event.GetManager().GetLogger(input.EventID)
+	defer event.GetManager().ReleaseLogger(logger)
+	defer func() {
+		if r := recover(); r != nil {
+			debug.PrintStack()
+			logger.Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
+		}
+	}()
 	logger.Info("开始应用构建源检测", map[string]string{"step": "starting"})
 	logrus.Infof("start check service by type: %s ", input.SourceType)
 	var pr parser.Parser