[REV] Support batch import applications.

api/controller/app.go

@@ -5,16 +5,13 @@ import (
 	"net/http"
 
 	"io"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/go-chi/chi"
 	"github.com/goodrain/rainbond/api/handler"
 	"github.com/goodrain/rainbond/api/model"
 	"github.com/goodrain/rainbond/db"
-	dbmodel "github.com/goodrain/rainbond/db/model"
 	httputil "github.com/goodrain/rainbond/util/http"
 )
 
@@ -35,7 +32,7 @@ func (a *AppStruct) ExportApp(w http.ResponseWriter, r *http.Request) {
 		}
 
 		// 要先更新数据库再通知builder组件
-		app := model.NewAppStatusFrom(&tr)
+		app := model.NewAppStatusFromExport(&tr)
 		db.GetManager().AppDao().DeleteModelByEventId(app.EventID)
 		if err := db.GetManager().AppDao().AddModel(app); err != nil {
 			httputil.ReturnError(r, w, 502, fmt.Sprintf("Failed to export app %s: %v", app.EventID, err))
@@ -110,40 +107,27 @@ func (a *AppStruct) Upload(w http.ResponseWriter, r *http.Request) {
 func (a *AppStruct) ImportApp(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
 	case "POST":
-		var app = dbmodel.AppStatus{
+		var importApp = model.ImportAppStruct{
 			Format: "rainbond-app",
-			Status: "importing",
 		}
 
-		ok := httputil.ValidatorRequestStructAndErrorResponse(r, w, &app, nil)
+		ok := httputil.ValidatorRequestStructAndErrorResponse(r, w, &importApp, nil)
 		if !ok {
 			return
 		}
 
 		// 获取tar包所在目录
-		sourceDir := fmt.Sprintf("%s/import/%s", handler.GetAppHandler().GetStaticDir(), app.EventID)
-		files, err := ioutil.ReadDir(sourceDir)
-		if err != nil {
-			httputil.ReturnError(r, w, 500, err.Error())
-			return
-		}
-
-		// 获取tar包的不含后缀的文件名
-		name := files[0].Name()
-		ex := filepath.Ext(files[0].Name())
-		name = files[0].Name()[:len(name)-len(ex)]
-
-		// tar包解压后的目录
-		app.SourceDir = fmt.Sprintf("%s/%s", sourceDir, name)
+		importApp.SourceDir = fmt.Sprintf("%s/import/%s", handler.GetAppHandler().GetStaticDir(), importApp.EventID)
 
 		// 要先更新数据库再通知builder组件
+		app := model.NewAppStatusFromImport(&importApp)
 		db.GetManager().AppDao().DeleteModelByEventId(app.EventID)
-		if err := db.GetManager().AppDao().AddModel(&app); err != nil {
-			httputil.ReturnError(r, w, 502, fmt.Sprintf("Failed to import app %s: %v", name, err))
+		if err := db.GetManager().AppDao().AddModel(app); err != nil {
+			httputil.ReturnError(r, w, 502, fmt.Sprintf("Failed to import app %s: %v", app.SourceDir, err))
 			return
 		}
 
-		err = handler.GetAppHandler().ImportApp(&app)
+		err := handler.GetAppHandler().ImportApp(&importApp)
 		if err != nil {
 			httputil.ReturnError(r, w, 501, fmt.Sprintf("Failed to import app: %v", err))
 			return

api/handler/app.go

@@ -8,7 +8,6 @@ import (
 	"github.com/goodrain/rainbond/api/model"
 	"github.com/goodrain/rainbond/api/util"
 	"github.com/goodrain/rainbond/mq/api/grpc/pb"
-	dbmodel "github.com/goodrain/rainbond/db/model"
 	"io/ioutil"
 	"os"
 	"fmt"
@@ -93,8 +92,8 @@ func (a *AppAction) ExportApp(tr *model.ExportAppStruct) error {
 	return nil
 }
 
-func (a *AppAction) ImportApp(app *dbmodel.AppStatus) error {
-	mqBody, err := json.Marshal(app)
+func (a *AppAction) ImportApp(importApp *model.ImportAppStruct) error {
+	mqBody, err := json.Marshal(importApp)
 
 	ts := &db.BuildTaskStruct{
 		TaskType: "import_app",

api/model/model.go

@@ -1357,7 +1357,7 @@ type ExportAppStruct struct {
 		EventID       string `json:"event_id"`
 		GroupKey      string `json:"group_key"` // TODO 考虑去掉
 		Version       string `json:"version"`   // TODO 考虑去掉
-		Format        string `json:"format"` // only rainbond-app/docker-compose
+		Format        string `json:"format"`    // only rainbond-app/docker-compose
 		GroupMetadata string `json:"group_metadata"`
 	}
 }
@@ -1380,7 +1380,7 @@ type MQBody struct {
 	SourceDir string `json:"source_dir"`
 }
 
-func NewAppStatusFrom(app *ExportAppStruct) *dbmodel.AppStatus {
+func NewAppStatusFromExport(app *ExportAppStruct) *dbmodel.AppStatus {
 	fields := strings.Split(app.SourceDir, "/")
 	tarName := fields[len(fields)-1]
 	tarFileHref := fmt.Sprintf("/v2/app/download/%s/%s.tar", app.Body.Format, tarName)
@@ -1392,3 +1392,35 @@ func NewAppStatusFrom(app *ExportAppStruct) *dbmodel.AppStatus {
 		TarFileHref: tarFileHref,
 	}
 }
+
+type ImportAppStruct struct {
+	EventID   string `json:"event_id"`
+	SourceDir string `json:"source_dir"`
+	Format    string `json:"format"`
+	ServiceImage ServiceImage `json:"service_image"`
+	ServiceSlug  ServiceSlug `json:"service_slug"`
+}
+
+type ServiceImage struct {
+	HubUrl      string `json:"hub_url"`
+	HubUser     string `json:"hub_user"`
+	HubPassword string `json:"hub_password"`
+	NameSpace   string `json:"namespace"`
+}
+
+type ServiceSlug struct {
+	FtpHost     string `json:"ftp_host"`
+	FtpPort     string `json:"ftp_port"`
+	FtpUsername string `json:"ftp_username"`
+	FtpPassword string `json:"ftp_password"`
+	NameSpace   string `json:"namespace"`
+}
+
+func NewAppStatusFromImport(app *ImportAppStruct) *dbmodel.AppStatus {
+	return &dbmodel.AppStatus{
+		EventID:   app.EventID,
+		Format:    app.Format,
+		SourceDir: app.SourceDir,
+		Status:    "importing",
+	}
+}

builder/exector/import_app.go

@@ -19,20 +19,22 @@
 package exector
 
 import (
+	"errors"
 	"fmt"
 	"github.com/Sirupsen/logrus"
+	"github.com/bitly/go-simplejson"
+	"github.com/docker/engine-api/client"
+	"github.com/goodrain/rainbond/builder/sources"
+	"github.com/goodrain/rainbond/db"
+	dbmodel "github.com/goodrain/rainbond/db/model"
+	"github.com/goodrain/rainbond/event"
+	"github.com/tidwall/gjson"
 	"io/ioutil"
 	"os/exec"
-	"strings"
 	"path/filepath"
-	"github.com/goodrain/rainbond/builder/sources"
-	"github.com/tidwall/gjson"
-	"github.com/goodrain/rainbond/event"
-	"github.com/docker/engine-api/client"
+	"strings"
 	"time"
-	"github.com/goodrain/rainbond/db"
-	"errors"
-	"github.com/goodrain/rainbond/db/model"
+	"github.com/goodrain/rainbond/api/model"
 )
 
 func init() {
@@ -44,6 +46,8 @@ type ImportApp struct {
 	EventID      string `json:"event_id"`
 	Format       string `json:"format"`
 	SourceDir    string `json:"source_dir"`
+	ServiceImage model.ServiceImage
+	ServiceSlug  model.ServiceSlug
 	Logger       event.Logger
 	DockerClient *client.Client
 }
@@ -57,10 +61,15 @@ func NewImportApp(in []byte) TaskWorker {
 	}
 
 	eventID := gjson.GetBytes(in, "event_id").String()
+	serviceImage := gjson.GetBytes(in, "service_image").Value().(model.ServiceImage)
+	serviceSlug := gjson.GetBytes(in, "service_image").Value().(model.ServiceSlug)
 	logger := event.GetManager().GetLogger(eventID)
+
 	return &ImportApp{
 		Format:       gjson.GetBytes(in, "format").String(),
 		SourceDir:    gjson.GetBytes(in, "source_dir").String(),
+		ServiceImage: serviceImage,
+		ServiceSlug:  serviceSlug,
 		Logger:       logger,
 		EventID:      eventID,
 		DockerClient: dockerClient,
@@ -103,12 +112,42 @@ func (i *ImportApp) importApp() error {
 		return err
 	}
 
-	// 上传镜像和源码包到仓库中
-	if err := i.loadApps(); err != nil {
+	// 如果目录下有多个子目录，则认为每个子目录是一个应用组，循环导入之
+	dirs, err := ioutil.ReadDir(i.SourceDir)
+	if err != nil {
+		logrus.Error("Failed to read dir for import app: ", i.SourceDir)
 		return err
 	}
 
-	// 更橷应用状态
+	oldSourceDir := i.SourceDir
+	var errMsg string
+	for _, dir := range dirs {
+		if dir.IsDir() {
+			continue
+		}
+
+		i.SourceDir = filepath.Join(i.SourceDir, dir.Name())
+
+		// 修改json元数据中的镜像和源码包仓库地址为指定地址
+		if err := i.replaceRepo(); err != nil {
+			logrus.Errorf("Failed to change repo address in metadata json for %s: %s", i.SourceDir, err)
+			errMsg = fmt.Sprintf("%s; %s", errMsg, err.Error())
+			continue
+		}
+
+		// 上传镜像和源码包到仓库中
+		if err := i.loadApps(); err != nil {
+			logrus.Errorf("Failed to load apps for %s: %s", i.SourceDir, err)
+			errMsg = fmt.Sprintf("%s; %s", errMsg, err.Error())
+			continue
+		}
+	}
+	i.SourceDir = oldSourceDir
+	if errMsg != "" {
+		return errors.New(errMsg)
+	}
+
+	// 更新应用状态
 	if err := i.updateStatus("success"); err != nil {
 		return err
 	}
@@ -116,8 +155,9 @@ func (i *ImportApp) importApp() error {
 	return nil
 }
 
+// i.SourceDir = "/grdata/app/import/n7brv4/web-app"
 func (i *ImportApp) unzip() error {
-	cmd := fmt.Sprintf("cd %s && rm -rf %s && tar -xf %s.tar", filepath.Dir(i.SourceDir), i.SourceDir, i.SourceDir)
+	cmd := fmt.Sprintf("cd %s && tar -xf *.tar", i.SourceDir)
 	err := exec.Command("sh", "-c", cmd).Run()
 	if err != nil {
 		logrus.Error("Failed to unzip for import app: ", i.SourceDir, ".tar")
@@ -128,6 +168,45 @@ func (i *ImportApp) unzip() error {
 	return err
 }
 
+// 替换元数据中的镜像和源码包的仓库地址
+func (i *ImportApp) replaceRepo() error {
+	metaFile := fmt.Sprintf("%s/metadata.json", i.SourceDir)
+	logrus.Debug("Change image and slug repo address in: ", metaFile)
+
+	data, err := ioutil.ReadFile(metaFile)
+	logrus.Debug("old json: ", string(data))
+	meta, err := simplejson.NewJson(data)
+	if err != nil {
+		return err
+	}
+
+	apps, err := meta.Get("apps").Array()
+	if err != nil {
+		return err
+	}
+
+	for index := range apps {
+		app := meta.Get("apps").GetIndex(index)
+		if _, ok := app.CheckGet("service_image"); ok {
+			app.Set("service_image", i.ServiceImage)
+		}
+
+		if _, ok := app.CheckGet("service_slug"); ok {
+			app.Set("service_slug", i.ServiceSlug)
+		}
+		apps[index] = app
+	}
+
+	meta.Set("apps", apps)
+	data, err = meta.Bytes()
+	if err != nil {
+		return err
+	}
+	logrus.Debug("new json: ", string(data))
+
+	return ioutil.WriteFile(metaFile, data, 0644)
+}
+
 //parseApps get apps array from metadata.json
 func (i *ImportApp) parseApps() ([]gjson.Result, error) {
 	i.Logger.Info("解析应用信息", map[string]string{"step": "export-app", "status": "success"})
@@ -244,7 +323,7 @@ func (i *ImportApp) updateStatus(status string) error {
 	}
 
 	// 在数据库中更新该应用的状态信息
-	app := res.(*model.AppStatus)
+	app := res.(*dbmodel.AppStatus)
 	app.Status = status
 	app.Metadata = string(data)