Merge branch 'master' into third-component

api/api/api_interface.go

@@ -53,6 +53,7 @@ type TenantInterface interface {
 	LimitTenantMemory(w http.ResponseWriter, r *http.Request)
 	TenantResourcesStatus(w http.ResponseWriter, r *http.Request)
 	CheckResourceName(w http.ResponseWriter, r *http.Request)
+	Log(w http.ResponseWriter, r *http.Request)
 }
 
 //ServiceInterface ServiceInterface

api/api_routers/version2/v2Routers.go

@@ -305,6 +305,8 @@ func (v2 *V2) serviceRouter() chi.Router {
 	r.Put("/service-monitors/{name}", middleware.WrapEL(controller.GetManager().UpdateServiceMonitors, dbmodel.TargetTypeService, "update-app-service-monitor", dbmodel.SYNEVENTTYPE))
 	r.Delete("/service-monitors/{name}", middleware.WrapEL(controller.GetManager().DeleteServiceMonitors, dbmodel.TargetTypeService, "delete-app-service-monitor", dbmodel.SYNEVENTTYPE))
 
+	r.Get("/log", controller.GetManager().Log)
+
 	return r
 }
 

api/controller/application.go

@@ -151,6 +151,11 @@ func (a *ApplicationController) ListComponents(w http.ResponseWriter, r *http.Re
 func (a *ApplicationController) DeleteApp(w http.ResponseWriter, r *http.Request) {
 	app := r.Context().Value(ctxutil.ContextKey("application")).(*dbmodel.Application)
 
+	var req model.EtcdCleanReq
+	if httputil.ValidatorRequestStructAndErrorResponse(r, w, &req, nil) {
+		logrus.Debugf("delete app etcd keys : %+v", req.Keys)
+		handler.GetEtcdHandler().CleanAllServiceData(req.Keys)
+	}
 	// Delete application
 	err := handler.GetApplicationHandler().DeleteApp(r.Context(), app)
 	if err != nil {

api/controller/service_action.go

@@ -23,8 +23,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-
+	"strconv"
-	validator "github.com/goodrain/rainbond/util/govalidator"
 
 	"github.com/go-chi/chi"
 	"github.com/goodrain/rainbond/api/handler"
@@ -33,6 +32,7 @@ import (
 	"github.com/goodrain/rainbond/db"
 	dbmodel "github.com/goodrain/rainbond/db/model"
 	"github.com/goodrain/rainbond/event"
+	validator "github.com/goodrain/rainbond/util/govalidator"
 	httputil "github.com/goodrain/rainbond/util/http"
 	"github.com/goodrain/rainbond/worker/discover/model"
 	"github.com/jinzhu/gorm"
@@ -774,3 +774,17 @@ func GetServiceDeployInfo(w http.ResponseWriter, r *http.Request) {
 	}
 	httputil.ReturnSuccess(r, w, info)
 }
+
+// Log -
+func (t *TenantStruct) Log(w http.ResponseWriter, r *http.Request) {
+	component := r.Context().Value(ctxutil.ContextKey("service")).(*dbmodel.TenantServices)
+	podName := r.URL.Query().Get("podName")
+	containerName := r.URL.Query().Get("containerName")
+	follow, _ := strconv.ParseBool(r.URL.Query().Get("follow"))
+
+	err := handler.GetServiceManager().Log(w, r, component, podName, containerName, follow)
+	if err != nil {
+		httputil.ReturnBcodeError(r, w, err)
+		return
+	}
+}

api/controller/volume.go

@@ -25,6 +25,7 @@ import (
 	"github.com/go-chi/chi"
 	"github.com/goodrain/rainbond/api/handler"
 	api_model "github.com/goodrain/rainbond/api/model"
+	"github.com/goodrain/rainbond/api/util/bcode"
 	ctxutil "github.com/goodrain/rainbond/api/util/ctx"
 	dbmodel "github.com/goodrain/rainbond/db/model"
 	httputil "github.com/goodrain/rainbond/util/http"
@@ -186,6 +187,11 @@ func (t *TenantStruct) UpdVolume(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if req.Mode != nil && (*req.Mode > 777 || *req.Mode < 0) {
+		httputil.ReturnBcodeError(r, w, bcode.NewBadRequest("mode be a number between 0 and 777 (octal)"))
+		return
+	}
+
 	sid := r.Context().Value(ctxutil.ContextKey("service_id")).(string)
 	if err := handler.GetServiceManager().UpdVolume(sid, &req); err != nil {
 		httputil.ReturnError(r, w, 500, err.Error())
@@ -354,6 +360,11 @@ func AddVolume(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if avs.Body.Mode != nil && (*avs.Body.Mode > 777 || *avs.Body.Mode < 0) {
+		httputil.ReturnBcodeError(r, w, bcode.NewBadRequest("mode be a number between 0 and 777 (octal)"))
+		return
+	}
+
 	tsv := &dbmodel.TenantServiceVolume{
 		ServiceID:          serviceID,
 		VolumeName:         avs.Body.VolumeName,
@@ -368,6 +379,7 @@ func AddVolume(w http.ResponseWriter, r *http.Request) {
 		BackupPolicy:       avs.Body.BackupPolicy,
 		ReclaimPolicy:      avs.Body.ReclaimPolicy,
 		AllowExpansion:     avs.Body.AllowExpansion,
+		Mode:               avs.Body.Mode,
 	}
 
 	// TODO fanyangyang validate VolumeCapacity  AccessMode SharePolicy BackupPolicy ReclaimPolicy AllowExpansion

api/handler/application_handler.go

@@ -572,6 +572,9 @@ func (a *ApplicationAction) SyncComponents(app *dbmodel.Application, components
 		if err := GetGatewayHandler().SyncHTTPRules(tx, components); err != nil {
 			return err
 		}
+		if err := GetGatewayHandler().SyncRuleConfigs(tx, components); err != nil {
+			return err
+		}
 		if err := GetGatewayHandler().SyncTCPRules(tx, components); err != nil {
 			return err
 		}

api/handler/gateway_action.go

@@ -885,8 +885,9 @@ func (g *GatewayAction) listHTTPRuleIDs(componentID string, port int) ([]string,
 // SyncHTTPRules -
 func (g *GatewayAction) SyncHTTPRules(tx *gorm.DB, components []*apimodel.Component) error {
 	var (
-		componentIDs []string
+		componentIDs   []string
-		httpRules    []*model.HTTPRule
+		httpRules      []*model.HTTPRule
+		ruleExtensions []*model.RuleExtension
 	)
 	for _, component := range components {
 		if component.HTTPRules == nil {
@@ -895,14 +896,40 @@ func (g *GatewayAction) SyncHTTPRules(tx *gorm.DB, components []*apimodel.Compon
 		componentIDs = append(componentIDs, component.ComponentBase.ComponentID)
 		for _, httpRule := range component.HTTPRules {
 			httpRules = append(httpRules, httpRule.DbModel(component.ComponentBase.ComponentID))
+
+			for _, ext := range httpRule.RuleExtensions {
+				ruleExtensions = append(ruleExtensions, &model.RuleExtension{
+					UUID:   util.NewUUID(),
+					RuleID: httpRule.HTTPRuleID,
+					Key:    ext.Key,
+					Value:  ext.Value,
+				})
+			}
 		}
 	}
+
+	if err := g.syncRuleExtensions(tx, httpRules, ruleExtensions); err != nil {
+		return err
+	}
+
 	if err := db.GetManager().HTTPRuleDaoTransactions(tx).DeleteByComponentIDs(componentIDs); err != nil {
 		return err
 	}
 	return db.GetManager().HTTPRuleDaoTransactions(tx).CreateOrUpdateHTTPRuleInBatch(httpRules)
 }
 
+func (g *GatewayAction) syncRuleExtensions(tx *gorm.DB, httpRules []*model.HTTPRule, exts []*model.RuleExtension) error {
+	var ruleIDs []string
+	for _, hr := range httpRules {
+		ruleIDs = append(ruleIDs, hr.UUID)
+	}
+
+	if err := db.GetManager().RuleExtensionDaoTransactions(tx).DeleteByRuleIDs(ruleIDs); err != nil {
+		return err
+	}
+	return db.GetManager().RuleExtensionDaoTransactions(tx).CreateOrUpdateRuleExtensionsInBatch(exts)
+}
+
 // SyncTCPRules -
 func (g *GatewayAction) SyncTCPRules(tx *gorm.DB, components []*apimodel.Component) error {
 	var (
@@ -923,3 +950,34 @@ func (g *GatewayAction) SyncTCPRules(tx *gorm.DB, components []*apimodel.Compone
 	}
 	return db.GetManager().TCPRuleDaoTransactions(tx).CreateOrUpdateTCPRuleInBatch(tcpRules)
 }
+
+// SyncRuleConfigs -
+func (g *GatewayAction) SyncRuleConfigs(tx *gorm.DB, components []*apimodel.Component) error {
+	var configs []*model.GwRuleConfig
+	var componentIDs []string
+	for _, component := range components {
+		componentIDs = append(componentIDs, component.ComponentBase.ComponentID)
+		if len(component.HTTPRuleConfigs) == 0 {
+			continue
+		}
+
+		for _, httpRuleConfig := range component.HTTPRuleConfigs {
+			configs = append(configs, httpRuleConfig.DbModel()...)
+		}
+	}
+
+	// http rule ids
+	rules, err := db.GetManager().HTTPRuleDao().ListByComponentIDs(componentIDs)
+	if err != nil {
+		return err
+	}
+	var ruleIDs []string
+	for _, rule := range rules {
+		ruleIDs = append(ruleIDs, rule.UUID)
+	}
+
+	if err := db.GetManager().GwRuleConfigDaoTransactions(tx).DeleteByRuleIDs(ruleIDs); err != nil {
+		return err
+	}
+	return db.GetManager().GwRuleConfigDaoTransactions(tx).CreateOrUpdateGwRuleConfigsInBatch(configs)
+}

api/handler/gateway_handler.go

@@ -61,4 +61,5 @@ type GatewayHandler interface {
 	DeleteIngressRulesByComponentPort(tx *gorm.DB, componentID string, port int) error
 	SyncHTTPRules(tx *gorm.DB, components []*apimodel.Component) error
 	SyncTCPRules(tx *gorm.DB, components []*apimodel.Component) error
+	SyncRuleConfigs(tx *gorm.DB, components []*apimodel.Component) error
 }

api/handler/handler.go

@@ -60,7 +60,7 @@ func InitHandle(conf option.Config,
 		return err
 	}
 	dbmanager := db.GetManager()
-	defaultServieHandler = CreateManager(conf, mqClient, etcdcli, statusCli, prometheusCli, rainbondClient)
+	defaultServieHandler = CreateManager(conf, mqClient, etcdcli, statusCli, prometheusCli, rainbondClient, kubeClient)
 	defaultPluginHandler = CreatePluginManager(mqClient)
 	defaultAppHandler = CreateAppManager(mqClient)
 	defaultTenantHandler = CreateTenManager(mqClient, statusCli, &conf, kubeClient, prometheusCli, k8sClient)

api/handler/service.go

@@ -22,6 +22,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -29,14 +31,20 @@ import (
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/goodrain/rainbond/api/client/prometheus"
+	api_model "github.com/goodrain/rainbond/api/model"
 	"github.com/goodrain/rainbond/api/util"
 	"github.com/goodrain/rainbond/api/util/bcode"
 	"github.com/goodrain/rainbond/api/util/license"
 	"github.com/goodrain/rainbond/builder/parser"
 	"github.com/goodrain/rainbond/cmd/api/option"
 	"github.com/goodrain/rainbond/db"
+	dberr "github.com/goodrain/rainbond/db/errors"
+	dbmodel "github.com/goodrain/rainbond/db/model"
 	"github.com/goodrain/rainbond/event"
+	gclient "github.com/goodrain/rainbond/mq/client"
 	"github.com/goodrain/rainbond/pkg/generated/clientset/versioned"
+	core_util "github.com/goodrain/rainbond/util"
+	typesv1 "github.com/goodrain/rainbond/worker/appm/types/v1"
 	"github.com/goodrain/rainbond/worker/client"
 	"github.com/goodrain/rainbond/worker/discover/model"
 	"github.com/goodrain/rainbond/worker/server"
@@ -46,17 +54,11 @@ import (
 	"github.com/pquerna/ffjson/ffjson"
 	"github.com/sirupsen/logrus"
 	"github.com/twinj/uuid"
+	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
+	"k8s.io/apiserver/pkg/util/flushwriter"
-	api_model "github.com/goodrain/rainbond/api/model"
+	"k8s.io/client-go/kubernetes"
-	dberr "github.com/goodrain/rainbond/db/errors"
-	core_model "github.com/goodrain/rainbond/db/model"
-	dbmodel "github.com/goodrain/rainbond/db/model"
-	eventutil "github.com/goodrain/rainbond/eventlog/util"
-	gclient "github.com/goodrain/rainbond/mq/client"
-	core_util "github.com/goodrain/rainbond/util"
-	typesv1 "github.com/goodrain/rainbond/worker/appm/types/v1"
 )
 
 // ErrServiceNotClosed -
@@ -70,6 +72,7 @@ type ServiceAction struct {
 	prometheusCli  prometheus.Interface
 	conf           option.Config
 	rainbondClient versioned.Interface
+	kubeClient     kubernetes.Interface
 }
 
 type dCfg struct {
@@ -86,7 +89,8 @@ func CreateManager(conf option.Config,
 	etcdCli *clientv3.Client,
 	statusCli *client.AppRuntimeSyncClient,
 	prometheusCli prometheus.Interface,
-	rainbondClient versioned.Interface) *ServiceAction {
+	rainbondClient versioned.Interface,
+	kubeClient kubernetes.Interface) *ServiceAction {
 	return &ServiceAction{
 		MQClient:       mqClient,
 		EtcdCli:        etcdCli,
@@ -94,6 +98,7 @@ func CreateManager(conf option.Config,
 		conf:           conf,
 		prometheusCli:  prometheusCli,
 		rainbondClient: rainbondClient,
+		kubeClient:     kubeClient,
 	}
 }
 
@@ -713,10 +718,10 @@ func (s *ServiceAction) ServiceCreate(sc *api_model.ServiceStruct) error {
 	if sc.OSType == "windows" {
 		if err := db.GetManager().TenantServiceLabelDaoTransactions(tx).AddModel(&dbmodel.TenantServiceLable{
 			ServiceID:  ts.ServiceID,
-			LabelKey:   core_model.LabelKeyNodeSelector,
+			LabelKey:   dbmodel.LabelKeyNodeSelector,
 			LabelValue: sc.OSType,
 		}); err != nil {
-			logrus.Errorf("add label %s=%s  %v error, %v", core_model.LabelKeyNodeSelector, sc.OSType, ts.ServiceID, err)
+			logrus.Errorf("add label %s=%s  %v error, %v", dbmodel.LabelKeyNodeSelector, sc.OSType, ts.ServiceID, err)
 			tx.Rollback()
 			return err
 		}
@@ -1743,6 +1748,7 @@ func (s *ServiceAction) UpdVolume(sid string, req *api_model.UpdVolumeReq) error
 		return err
 	}
 	v.VolumePath = req.VolumePath
+	v.Mode = req.Mode
 	if err := db.GetManager().TenantServiceVolumeDaoTransactions(tx).UpdateModel(v); err != nil {
 		tx.Rollback()
 		return err
@@ -1937,7 +1943,7 @@ func (s *ServiceAction) GetStatus(serviceID string) (*api_model.StatusList, erro
 
 //GetServicesStatus  获取一组应用状态，若 serviceIDs为空,获取租户所有应用状态
 func (s *ServiceAction) GetServicesStatus(tenantID string, serviceIDs []string) []map[string]interface{} {
-	if serviceIDs == nil || len(serviceIDs) == 0 {
+	if len(serviceIDs) == 0 {
 		services, _ := db.GetManager().TenantServiceDao().GetServicesByTenantID(tenantID)
 		for _, s := range services {
 			serviceIDs = append(serviceIDs, s.ServiceID)
@@ -1948,11 +1954,9 @@ func (s *ServiceAction) GetServicesStatus(tenantID string, serviceIDs []string)
 	}
 	statusList := s.statusCli.GetStatuss(strings.Join(serviceIDs, ","))
 	var info = make([]map[string]interface{}, 0)
-	if statusList != nil {
+	for k, v := range statusList {
-		for k, v := range statusList {
+		serviceInfo := map[string]interface{}{"service_id": k, "status": v, "status_cn": TransStatus(v), "used_mem": 0}
-			serviceInfo := map[string]interface{}{"service_id": k, "status": v, "status_cn": TransStatus(v), "used_mem": 0}
+		info = append(info, serviceInfo)
-			info = append(info, serviceInfo)
-		}
 	}
 	return info
 }
@@ -2017,7 +2021,7 @@ func (s *ServiceAction) CreateTenant(t *dbmodel.Tenants) error {
 
 //CreateTenandIDAndName create tenant_id and tenant_name
 func (s *ServiceAction) CreateTenandIDAndName(eid string) (string, string, error) {
-	id := fmt.Sprintf("%s", uuid.NewV4())
+	id := uuid.NewV4().String()
 	uid := strings.Replace(id, "-", "", -1)
 	name := strings.Split(id, "-")[0]
 	logrus.Debugf("uuid is %v, name is %v", uid, name)
@@ -2101,14 +2105,12 @@ func (s *ServiceAction) GetMultiServicePods(serviceIDs []string) (*K8sPodInfos,
 	}
 	convpod := func(serviceID string, pods []*pb.ServiceAppPod) []*K8sPodInfo {
 		var podsInfoList []*K8sPodInfo
-		var podNames []string
 		for _, v := range pods {
 			var podInfo K8sPodInfo
 			podInfo.PodName = v.PodName
 			podInfo.PodIP = v.PodIp
 			podInfo.PodStatus = v.PodStatus
 			podInfo.ServiceID = serviceID
-			podNames = append(podNames, v.PodName)
 			podsInfoList = append(podsInfoList, &podInfo)
 		}
 		return podsInfoList
@@ -2296,23 +2298,6 @@ func (s *ServiceAction) deleteThirdComponent(ctx context.Context, component *dbm
 	return nil
 }
 
-// delLogFile deletes persistent data related to the service based on serviceID.
-func (s *ServiceAction) delLogFile(serviceID string, eventIDs []string) {
-	// log generated during service running
-	dockerLogPath := eventutil.DockerLogFilePath(s.conf.LogPath, serviceID)
-	if err := os.RemoveAll(dockerLogPath); err != nil {
-		logrus.Warningf("remove docker log files: %v", err)
-	}
-	// log generated by the service event
-	eventLogPath := eventutil.EventLogFilePath(s.conf.LogPath)
-	for _, eventID := range eventIDs {
-		eventLogFileName := eventutil.EventLogFileName(eventLogPath, eventID)
-		if err := os.RemoveAll(eventLogFileName); err != nil {
-			logrus.Warningf("file: %s; remove event log file: %v", eventLogFileName, err)
-		}
-	}
-}
-
 func (s *ServiceAction) gcTaskBody(tenantID, serviceID string) (map[string]interface{}, error) {
 	events, err := db.GetManager().ServiceEventDao().ListByTargetID(serviceID)
 	if err != nil {
@@ -2900,6 +2885,49 @@ func (s *ServiceAction) SyncComponentEndpoints(tx *gorm.DB, components []*api_mo
 	return db.GetManager().ThirdPartySvcDiscoveryCfgDaoTransactions(tx).CreateOrUpdate3rdSvcDiscoveryCfgInBatch(thirdPartySvcDiscoveryCfgs)
 }
 
+// Log returns the logs reader for a container in a pod, a pod or a component.
+func (s *ServiceAction) Log(w http.ResponseWriter, r *http.Request, component *dbmodel.TenantServices, podName, containerName string, follow bool) error {
+	// If podName and containerName is missing, return the logs reader for the component
+	// If containerName is missing, return the logs reader for the pod.
+	if podName == "" || containerName == "" {
+		// Only support return the logs reader for a container now.
+		return errors.WithStack(bcode.NewBadRequest("the field 'podName' and 'containerName' is required"))
+	}
+
+	request := s.kubeClient.CoreV1().Pods(component.TenantID).GetLogs(podName, &corev1.PodLogOptions{
+		Container: containerName,
+		Follow:    follow,
+	})
+
+	out, err := request.Stream(context.TODO())
+	if err != nil {
+		if k8sErrors.IsNotFound(err) {
+			return errors.Wrap(bcode.ErrPodNotFound, "get pod log")
+		}
+		return errors.Wrap(err, "get stream from request")
+	}
+	defer out.Close()
+
+	w.Header().Set("Transfer-Encoding", "chunked")
+	w.WriteHeader(http.StatusOK)
+
+	// Flush headers, if possible
+	if flusher, ok := w.(http.Flusher); ok {
+		flusher.Flush()
+	}
+
+	writer := flushwriter.Wrap(w)
+
+	_, err = io.Copy(writer, out)
+	if err != nil {
+		if strings.HasSuffix(err.Error(), "write: broken pipe") {
+			return nil
+		}
+		logrus.Warningf("write stream to response: %v", err)
+	}
+	return nil
+}
+
 //TransStatus trans service status
 func TransStatus(eStatus string) string {
 	switch eStatus {
@@ -2928,25 +2956,3 @@ func TransStatus(eStatus string) string {
 	}
 	return ""
 }
-
-//CheckLabel check label
-func CheckLabel(serviceID string) bool {
-	//true for v2, false for v1
-	serviceLabel, err := db.GetManager().TenantServiceLabelDao().GetTenantServiceLabel(serviceID)
-	if err != nil {
-		return false
-	}
-	if serviceLabel != nil && len(serviceLabel) > 0 {
-		return true
-	}
-	return false
-}
-
-//CheckMapKey CheckMapKey
-func CheckMapKey(rebody map[string]interface{}, key string, defaultValue interface{}) map[string]interface{} {
-	if _, ok := rebody[key]; ok {
-		return rebody
-	}
-	rebody[key] = defaultValue
-	return rebody
-}

api/handler/service_handler.go

@@ -20,7 +20,7 @@ package handler
 
 import (
 	"context"
-	"github.com/jinzhu/gorm"
+	"net/http"
 
 	api_model "github.com/goodrain/rainbond/api/model"
 	"github.com/goodrain/rainbond/api/util"
@@ -28,6 +28,7 @@ import (
 	dbmodel "github.com/goodrain/rainbond/db/model"
 	"github.com/goodrain/rainbond/worker/discover/model"
 	"github.com/goodrain/rainbond/worker/server/pb"
+	"github.com/jinzhu/gorm"
 )
 
 //ServiceHandler service handler
@@ -90,16 +91,18 @@ type ServiceHandler interface {
 	AddServiceMonitor(tenantID, serviceID string, add api_model.AddServiceMonitorRequestStruct) (*dbmodel.TenantServiceMonitor, error)
 
 	SyncComponentBase(tx *gorm.DB, app *dbmodel.Application, components []*api_model.Component) error
-	SyncComponentMonitors(tx *gorm.DB,app *dbmodel.Application, components []*api_model.Component) error
+	SyncComponentMonitors(tx *gorm.DB, app *dbmodel.Application, components []*api_model.Component) error
 	SyncComponentPorts(tx *gorm.DB, app *dbmodel.Application, components []*api_model.Component) error
 	SyncComponentRelations(tx *gorm.DB, app *dbmodel.Application, components []*api_model.Component) error
 	SyncComponentEnvs(tx *gorm.DB, app *dbmodel.Application, components []*api_model.Component) error
 	SyncComponentVolumeRels(tx *gorm.DB, app *dbmodel.Application, components []*api_model.Component) error
-	SyncComponentVolumes(tx *gorm.DB,  components []*api_model.Component) error
+	SyncComponentVolumes(tx *gorm.DB, components []*api_model.Component) error
-	SyncComponentConfigFiles(tx *gorm.DB,  components []*api_model.Component) error
+	SyncComponentConfigFiles(tx *gorm.DB, components []*api_model.Component) error
-	SyncComponentProbes(tx *gorm.DB,  components []*api_model.Component) error
+	SyncComponentProbes(tx *gorm.DB, components []*api_model.Component) error
-	SyncComponentLabels(tx *gorm.DB,  components []*api_model.Component) error
+	SyncComponentLabels(tx *gorm.DB, components []*api_model.Component) error
 	SyncComponentPlugins(tx *gorm.DB, app *dbmodel.Application, components []*api_model.Component) error
-	SyncComponentScaleRules(tx *gorm.DB,  components []*api_model.Component) error
+	SyncComponentScaleRules(tx *gorm.DB, components []*api_model.Component) error
 	SyncComponentEndpoints(tx *gorm.DB, components []*api_model.Component) error
+
+	Log(w http.ResponseWriter, r *http.Request, component *dbmodel.TenantServices, podName, containerName string, follow bool) error
 }

api/model/component.go

@@ -2,8 +2,9 @@ package model
 
 import (
 	"fmt"
-	dbmodel "github.com/goodrain/rainbond/db/model"
 	"time"
+
+	dbmodel "github.com/goodrain/rainbond/db/model"
 )
 
 // ComponentBase -
@@ -168,6 +169,7 @@ type ComponentVolume struct {
 	ReclaimPolicy      string `json:"reclaim_policy"`
 	AllowExpansion     bool   `json:"allow_expansion"`
 	VolumeProviderName string `json:"volume_provider_name"`
+	Mode               *int32 `json:"mode"`
 }
 
 // Key returns the key of ComponentVolume.
@@ -192,6 +194,7 @@ func (v *ComponentVolume) DbModel(componentID string) *dbmodel.TenantServiceVolu
 		ReclaimPolicy:      v.ReclaimPolicy,
 		AllowExpansion:     v.AllowExpansion,
 		VolumeProviderName: v.VolumeProviderName,
+		Mode:               v.Mode,
 	}
 }
 
@@ -239,6 +242,7 @@ type Component struct {
 	ComponentBase      ComponentBase                    `json:"component_base"`
 	HTTPRules          []AddHTTPRuleStruct              `json:"http_rules"`
 	TCPRules           []AddTCPRuleStruct               `json:"tcp_rules"`
+	HTTPRuleConfigs    []HTTPRuleConfig                 `json:"http_rule_configs"`
 	Monitors           []AddServiceMonitorRequestStruct `json:"monitors"`
 	Ports              []TenantServicesPort             `json:"ports"`
 	Relations          []TenantComponentRelation        `json:"relations"`

api/model/gateway_model.go

@@ -19,8 +19,10 @@
 package model
 
 import (
-	dbmodel "github.com/goodrain/rainbond/db/model"
+	"strconv"
 	"strings"
+
+	dbmodel "github.com/goodrain/rainbond/db/model"
 )
 
 //AddHTTPRuleStruct is used to add http rule, certificate and rule extensions
@@ -171,6 +173,79 @@ type Body struct {
 	ProxyBuffering      string       `json:"proxy_buffering,omitempty" validate:"proxy_buffering|required"`
 }
 
+// HTTPRuleConfig -
+type HTTPRuleConfig struct {
+	RuleID              string       `json:"rule_id,omitempty" validate:"rule_id|required"`
+	ProxyConnectTimeout int          `json:"proxy_connect_timeout,omitempty" validate:"proxy_connect_timeout|required"`
+	ProxySendTimeout    int          `json:"proxy_send_timeout,omitempty" validate:"proxy_send_timeout|required"`
+	ProxyReadTimeout    int          `json:"proxy_read_timeout,omitempty" validate:"proxy_read_timeout|required"`
+	ProxyBodySize       int          `json:"proxy_body_size,omitempty" validate:"proxy_body_size|required"`
+	SetHeaders          []*SetHeader `json:"set_headers,omitempty" `
+	Rewrites            []*Rewrite   `json:"rewrite,omitempty"`
+	ProxyBufferSize     int          `json:"proxy_buffer_size,omitempty" validate:"proxy_buffer_size|numeric_between:1,65535"`
+	ProxyBufferNumbers  int          `json:"proxy_buffer_numbers,omitempty" validate:"proxy_buffer_size|numeric_between:1,65535"`
+	ProxyBuffering      string       `json:"proxy_buffering,omitempty" validate:"proxy_buffering|required"`
+}
+
+// DbModel return database model
+func (h *HTTPRuleConfig) DbModel() []*dbmodel.GwRuleConfig {
+	var configs []*dbmodel.GwRuleConfig
+	configs = append(configs, &dbmodel.GwRuleConfig{
+		RuleID: h.RuleID,
+		Key:    "proxy-connect-timeout",
+		Value:  strconv.Itoa(h.ProxyConnectTimeout),
+	})
+	configs = append(configs, &dbmodel.GwRuleConfig{
+		RuleID: h.RuleID,
+		Key:    "proxy-send-timeout",
+		Value:  strconv.Itoa(h.ProxySendTimeout),
+	})
+	configs = append(configs, &dbmodel.GwRuleConfig{
+		RuleID: h.RuleID,
+		Key:    "proxy-read-timeout",
+		Value:  strconv.Itoa(h.ProxyReadTimeout),
+	})
+	configs = append(configs, &dbmodel.GwRuleConfig{
+		RuleID: h.RuleID,
+		Key:    "proxy-body-size",
+		Value:  strconv.Itoa(h.ProxyBodySize),
+	})
+	configs = append(configs, &dbmodel.GwRuleConfig{
+		RuleID: h.RuleID,
+		Key:    "proxy-buffer-size",
+		Value:  strconv.Itoa(h.ProxyBufferSize),
+	})
+	configs = append(configs, &dbmodel.GwRuleConfig{
+		RuleID: h.RuleID,
+		Key:    "proxy-buffer-numbers",
+		Value:  strconv.Itoa(h.ProxyBufferNumbers),
+	})
+	configs = append(configs, &dbmodel.GwRuleConfig{
+		RuleID: h.RuleID,
+		Key:    "proxy-buffering",
+		Value:  h.ProxyBuffering,
+	})
+	setheaders := make(map[string]string)
+	for _, item := range h.SetHeaders {
+		if strings.TrimSpace(item.Key) == "" {
+			continue
+		}
+		if strings.TrimSpace(item.Value) == "" {
+			item.Value = "empty"
+		}
+		// filter same key
+		setheaders["set-header-"+item.Key] = item.Value
+	}
+	for k, v := range setheaders {
+		configs = append(configs, &dbmodel.GwRuleConfig{
+			RuleID: h.RuleID,
+			Key:    k,
+			Value:  v,
+		})
+	}
+	return configs
+}
+
 //SetHeader set header
 type SetHeader struct {
 	Key   string `json:"item_key"`

api/model/volume.go

@@ -60,7 +60,8 @@ type AddVolumeStruct struct {
 		// ReclaimPolicy 回收策略
 		ReclaimPolicy string `json:"reclaim_policy"`
 		// AllowExpansion 是否支持扩展
-		AllowExpansion bool `json:"allow_expansion"`
+		AllowExpansion bool   `json:"allow_expansion"`
+		Mode           *int32 `json:"mode"`
 	}
 }
 
@@ -243,6 +244,7 @@ type UpdVolumeReq struct {
 	VolumeType  string `json:"volume_type" validate:"volume_type|required"`
 	FileContent string `json:"file_content"`
 	VolumePath  string `json:"volume_path" validate:"volume_path|required"`
+	Mode        *int32 `json:"mode"`
 }
 
 // VolumeWithStatusResp volume status

api/util/bcode/service.go

@@ -12,4 +12,5 @@ var (
 	ErrSyncOperation = newByMessage(409, 10103, "The asynchronous operation is executing")
 	// ErrHorizontalDueToNoChange
 	ErrHorizontalDueToNoChange = newByMessage(400, 10104, "The number of components has not changed, no need to scale")
+	ErrPodNotFound             = newByMessage(404, 10105, "pod not found")
 )

builder/exector/export_app.go

@@ -70,6 +70,7 @@ func NewExportApp(in []byte, m *exectorManager) (TaskWorker, error) {
 
 //Run Run
 func (i *ExportApp) Run(timeout time.Duration) error {
+	defer os.RemoveAll(i.SourceDir)
 	// disable Md5 checksum
 	// if ok := i.isLatest(); ok {
 	// 	i.updateStatus("success")

builder/exector/groupapp_restore.go

@@ -118,6 +118,9 @@ func (b *BackupAPPRestore) Run(timeout time.Duration) error {
 	if err := util.CheckAndCreateDir(cacheDir); err != nil {
 		return fmt.Errorf("create cache dir error %s", err.Error())
 	}
+	// delete the cache data
+	defer os.RemoveAll(cacheDir)
+
 	b.cacheDir = cacheDir
 	switch backup.BackupMode {
 	case "full-online":

db/dao/dao.go

@@ -505,6 +505,7 @@ type RuleExtensionDao interface {
 	GetRuleExtensionByRuleID(ruleID string) ([]*model.RuleExtension, error)
 	DeleteRuleExtensionByRuleID(ruleID string) error
 	DeleteByRuleIDs(ruleIDs []string) error
+	CreateOrUpdateRuleExtensionsInBatch(exts []*model.RuleExtension) error
 }
 
 // HTTPRuleDao -
@@ -521,6 +522,7 @@ type HTTPRuleDao interface {
 	DeleteByComponentPort(componentID string, port int) error
 	DeleteByComponentIDs(componentIDs []string) error
 	CreateOrUpdateHTTPRuleInBatch(httpRules []*model.HTTPRule) error
+	ListByComponentIDs(componentIDs []string) ([]*model.HTTPRule, error)
 }
 
 // TCPRuleDao -
@@ -565,6 +567,7 @@ type GwRuleConfigDao interface {
 	DeleteByRuleID(rid string) error
 	ListByRuleID(rid string) ([]*model.GwRuleConfig, error)
 	DeleteByRuleIDs(ruleIDs []string) error
+	CreateOrUpdateGwRuleConfigsInBatch(ruleConfigs []*model.GwRuleConfig) error
 }
 
 // TenantServceAutoscalerRulesDao -

db/model/tenant.go

@@ -476,6 +476,7 @@ type TenantServiceVolume struct {
 	AllowExpansion bool `gorm:"column:allow_expansion" json:"allow_expansion"`
 	// VolumeProviderName 使用的存储驱动别名
 	VolumeProviderName string `gorm:"column:volume_provider_name" json:"volume_provider_name"`
+	Mode               *int32 `gorm:"column:mode" json:"mode"`
 }
 
 //TableName 表名

db/mysql/dao/gateway.go

@@ -20,9 +20,9 @@ package dao
 
 import (
 	"fmt"
-	gormbulkups "github.com/atcdot/gorm-bulk-upsert"
 	"reflect"
 
+	gormbulkups "github.com/atcdot/gorm-bulk-upsert"
 	"github.com/goodrain/rainbond/api/util/bcode"
 	"github.com/goodrain/rainbond/db/model"
 	"github.com/jinzhu/gorm"
@@ -149,6 +149,18 @@ func (c *RuleExtensionDaoImpl) DeleteByRuleIDs(ruleIDs []string) error {
 	return nil
 }
 
+// CreateOrUpdateRuleExtensionsInBatch -
+func (c *RuleExtensionDaoImpl) CreateOrUpdateRuleExtensionsInBatch(exts []*model.RuleExtension) error {
+	var objects []interface{}
+	for _, ext := range exts {
+		objects = append(objects, *ext)
+	}
+	if err := gormbulkups.BulkUpsert(c.DB, objects, 2000); err != nil {
+		return errors.Wrap(err, "create or update rule extensions in batch")
+	}
+	return nil
+}
+
 //HTTPRuleDaoImpl http rule
 type HTTPRuleDaoImpl struct {
 	DB *gorm.DB
@@ -276,7 +288,7 @@ func (h *HTTPRuleDaoImpl) ListByCertID(certID string) ([]*model.HTTPRule, error)
 }
 
 //DeleteByComponentIDs delete http rule by component ids
-func (h *HTTPRuleDaoImpl) DeleteByComponentIDs(componentIDs []string) error{
+func (h *HTTPRuleDaoImpl) DeleteByComponentIDs(componentIDs []string) error {
 	return h.DB.Where("service_id in (?) ", componentIDs).Delete(&model.HTTPRule{}).Error
 }
 
@@ -292,6 +304,15 @@ func (h *HTTPRuleDaoImpl) CreateOrUpdateHTTPRuleInBatch(httpRules []*model.HTTPR
 	return nil
 }
 
+// ListByComponentIDs -
+func (h *HTTPRuleDaoImpl) ListByComponentIDs(componentIDs []string) ([]*model.HTTPRule, error) {
+	var rules []*model.HTTPRule
+	if err := h.DB.Where("service_id in (?) ", componentIDs).Find(&rules).Error; err != nil {
+		return nil, err
+	}
+	return rules, nil
+}
+
 // TCPRuleDaoTmpl is a implementation of TcpRuleDao
 type TCPRuleDaoTmpl struct {
 	DB *gorm.DB
@@ -407,7 +428,7 @@ func (t *TCPRuleDaoTmpl) ListByServiceID(serviceID string) ([]*model.TCPRule, er
 }
 
 //DeleteByComponentIDs delete tcp rule by component ids
-func (t *TCPRuleDaoTmpl) DeleteByComponentIDs(componentIDs []string) error{
+func (t *TCPRuleDaoTmpl) DeleteByComponentIDs(componentIDs []string) error {
 	return t.DB.Where("service_id in (?) ", componentIDs).Delete(&model.TCPRule{}).Error
 }
 
@@ -470,3 +491,15 @@ func (t *GwRuleConfigDaoImpl) DeleteByRuleIDs(ruleIDs []string) error {
 	}
 	return nil
 }
+
+// CreateOrUpdateGwRuleConfigsInBatch creates or updates rule configs in batch.
+func (t *GwRuleConfigDaoImpl) CreateOrUpdateGwRuleConfigsInBatch(ruleConfigs []*model.GwRuleConfig) error {
+	var objects []interface{}
+	for _, ruleConfig := range ruleConfigs {
+		objects = append(objects, *ruleConfig)
+	}
+	if err := gormbulkups.BulkUpsert(t.DB, objects, 2000); err != nil {
+		return errors.Wrap(err, "create or update rule configs in batch")
+	}
+	return nil
+}

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/mock v1.4.4
 	github.com/golang/protobuf v1.4.3
-	github.com/goodrain/rainbond-oam v0.0.0-20210721020036-158e1be667dc
+	github.com/goodrain/rainbond-oam v0.0.0-20210810094229-f1cd639c451a
 	github.com/goodrain/rainbond-operator v1.3.1-0.20210401055914-f8fe4bf89a21
 	github.com/google/go-cmp v0.5.4 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect

go.sum

@@ -696,6 +696,8 @@ github.com/goodrain/gorm-bulk-upsert v1.0.1-0.20210608013724-7e7870d16357 h1:kdS
 github.com/goodrain/gorm-bulk-upsert v1.0.1-0.20210608013724-7e7870d16357/go.mod h1:b7/GgeVNbf/SFw4FYIWslxNV5I10C9Mhf/++3jsDk3M=
 github.com/goodrain/rainbond-oam v0.0.0-20210721020036-158e1be667dc h1:hCtxb/Yy4G+wEc2n+yaXx3j4SF/s34zNI8XK5qkHqXk=
 github.com/goodrain/rainbond-oam v0.0.0-20210721020036-158e1be667dc/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
+github.com/goodrain/rainbond-oam v0.0.0-20210810094229-f1cd639c451a h1:a48En+OrB5PzoOJflEEc77eCzh6mODRHkCpx+4kB2/0=
+github.com/goodrain/rainbond-oam v0.0.0-20210810094229-f1cd639c451a/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
 github.com/goodrain/rainbond-operator v1.3.1-0.20210401055914-f8fe4bf89a21 h1:iCPI96slvJv88iPc1NJW8zhpkiza0kwB0jtsuZIJLRQ=
 github.com/goodrain/rainbond-operator v1.3.1-0.20210401055914-f8fe4bf89a21/go.mod h1:jcQfNoxO67nkLalCmgihYrdWF82TKyuPW032tgGdqVY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=

worker/appm/volume/config-file.go

@@ -67,7 +67,7 @@ func (v *ConfigFileVolume) CreateVolume(define *Define) error {
 	}
 	cmap.Data[path.Base(v.svm.VolumePath)] = util.ParseVariable(cf.FileContent, configs)
 	v.as.SetConfigMap(cmap)
-	define.SetVolumeCMap(cmap, path.Base(v.svm.VolumePath), v.svm.VolumePath, false)
+	define.SetVolumeCMap(cmap, path.Base(v.svm.VolumePath), v.svm.VolumePath, false, v.svm.Mode)
 	return nil
 }
 
@@ -77,7 +77,7 @@ func (v *ConfigFileVolume) CreateDependVolume(define *Define) error {
 	for _, env := range v.envs {
 		configs[env.Name] = env.Value
 	}
-	_, err := v.dbmanager.TenantServiceVolumeDao().GetVolumeByServiceIDAndName(v.smr.DependServiceID, v.smr.VolumeName)
+	depVol, err := v.dbmanager.TenantServiceVolumeDao().GetVolumeByServiceIDAndName(v.smr.DependServiceID, v.smr.VolumeName)
 	if err != nil {
 		return fmt.Errorf("error getting TenantServiceVolume according to serviceID(%s) and volumeName(%s): %v",
 			v.smr.DependServiceID, v.smr.VolumeName, err)
@@ -98,6 +98,6 @@ func (v *ConfigFileVolume) CreateDependVolume(define *Define) error {
 	cmap.Data[path.Base(v.smr.VolumePath)] = util.ParseVariable(cf.FileContent, configs)
 	v.as.SetConfigMap(cmap)
 
-	define.SetVolumeCMap(cmap, path.Base(v.smr.VolumePath), v.smr.VolumePath, false)
+	define.SetVolumeCMap(cmap, path.Base(v.smr.VolumePath), v.smr.VolumePath, false, depVol.Mode)
 	return nil
 }

worker/appm/volume/volume.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/goodrain/rainbond/db"
@@ -211,8 +212,7 @@ func (v *Define) SetVolume(VolumeType dbmodel.VolumeType, name, mountPath, hostP
 }
 
 // SetVolumeCMap sets volumes and volumeMounts. The type of volumes is configMap.
-func (v *Define) SetVolumeCMap(cmap *corev1.ConfigMap, k, p string, isReadOnly bool) {
+func (v *Define) SetVolumeCMap(cmap *corev1.ConfigMap, k, p string, isReadOnly bool, mode *int32) {
-	var configFileMode int32 = 0777
 	vm := corev1.VolumeMount{
 		MountPath: p,
 		Name:      cmap.Name,
@@ -221,6 +221,11 @@ func (v *Define) SetVolumeCMap(cmap *corev1.ConfigMap, k, p string, isReadOnly b
 	}
 	v.volumeMounts = append(v.volumeMounts, vm)
 	var defaultMode int32 = 0777
+	if mode != nil {
+		// convert int to octal
+		octal, _ := strconv.ParseInt(strconv.Itoa(int(*mode)), 8, 64)
+		defaultMode = int32(octal)
+	}
 	vo := corev1.Volume{
 		Name: cmap.Name,
 		VolumeSource: corev1.VolumeSource{
@@ -233,7 +238,7 @@ func (v *Define) SetVolumeCMap(cmap *corev1.ConfigMap, k, p string, isReadOnly b
 					{
 						Key:  k,
 						Path: path.Base(p), // subpath
-						Mode: &configFileMode,
+						Mode: &defaultMode,
 					},
 				},
 			},