From ccf8df9f1d13f833be3d9a1b89b0af8ed95d9a5d Mon Sep 17 00:00:00 2001 From: Zhang Jiajun Date: Mon, 21 May 2018 18:21:42 +0800 Subject: [PATCH] [REV] Support batch import applications. --- api/controller/app.go | 32 +++-------- api/handler/app.go | 5 +- api/model/model.go | 36 +++++++++++- builder/exector/import_app.go | 105 +++++++++++++++++++++++++++++----- 4 files changed, 136 insertions(+), 42 deletions(-) diff --git a/api/controller/app.go b/api/controller/app.go index 945b4e8b6..8b4ff9696 100644 --- a/api/controller/app.go +++ b/api/controller/app.go @@ -5,16 +5,13 @@ import ( "net/http" "io" - "io/ioutil" "os" - "path/filepath" "strings" "github.com/go-chi/chi" "github.com/goodrain/rainbond/api/handler" "github.com/goodrain/rainbond/api/model" "github.com/goodrain/rainbond/db" - dbmodel "github.com/goodrain/rainbond/db/model" httputil "github.com/goodrain/rainbond/util/http" ) @@ -35,7 +32,7 @@ func (a *AppStruct) ExportApp(w http.ResponseWriter, r *http.Request) { } // 要先更新数据库再通知builder组件 - app := model.NewAppStatusFrom(&tr) + app := model.NewAppStatusFromExport(&tr) db.GetManager().AppDao().DeleteModelByEventId(app.EventID) if err := db.GetManager().AppDao().AddModel(app); err != nil { httputil.ReturnError(r, w, 502, fmt.Sprintf("Failed to export app %s: %v", app.EventID, err)) @@ -110,40 +107,27 @@ func (a *AppStruct) Upload(w http.ResponseWriter, r *http.Request) { func (a *AppStruct) ImportApp(w http.ResponseWriter, r *http.Request) { switch r.Method { case "POST": - var app = dbmodel.AppStatus{ + var importApp = model.ImportAppStruct{ Format: "rainbond-app", - Status: "importing", } - ok := httputil.ValidatorRequestStructAndErrorResponse(r, w, &app, nil) + ok := httputil.ValidatorRequestStructAndErrorResponse(r, w, &importApp, nil) if !ok { return } // 获取tar包所在目录 - sourceDir := fmt.Sprintf("%s/import/%s", handler.GetAppHandler().GetStaticDir(), app.EventID) - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - httputil.ReturnError(r, w, 500, err.Error()) - return - } - - // 获取tar包的不含后缀的文件名 - name := files[0].Name() - ex := filepath.Ext(files[0].Name()) - name = files[0].Name()[:len(name)-len(ex)] - - // tar包解压后的目录 - app.SourceDir = fmt.Sprintf("%s/%s", sourceDir, name) + importApp.SourceDir = fmt.Sprintf("%s/import/%s", handler.GetAppHandler().GetStaticDir(), importApp.EventID) // 要先更新数据库再通知builder组件 + app := model.NewAppStatusFromImport(&importApp) db.GetManager().AppDao().DeleteModelByEventId(app.EventID) - if err := db.GetManager().AppDao().AddModel(&app); err != nil { - httputil.ReturnError(r, w, 502, fmt.Sprintf("Failed to import app %s: %v", name, err)) + if err := db.GetManager().AppDao().AddModel(app); err != nil { + httputil.ReturnError(r, w, 502, fmt.Sprintf("Failed to import app %s: %v", app.SourceDir, err)) return } - err = handler.GetAppHandler().ImportApp(&app) + err := handler.GetAppHandler().ImportApp(&importApp) if err != nil { httputil.ReturnError(r, w, 501, fmt.Sprintf("Failed to import app: %v", err)) return diff --git a/api/handler/app.go b/api/handler/app.go index 45856dc83..c5c0382c0 100644 --- a/api/handler/app.go +++ b/api/handler/app.go @@ -8,7 +8,6 @@ import ( "github.com/goodrain/rainbond/api/model" "github.com/goodrain/rainbond/api/util" "github.com/goodrain/rainbond/mq/api/grpc/pb" - dbmodel "github.com/goodrain/rainbond/db/model" "io/ioutil" "os" "fmt" @@ -93,8 +92,8 @@ func (a *AppAction) ExportApp(tr *model.ExportAppStruct) error { return nil } -func (a *AppAction) ImportApp(app *dbmodel.AppStatus) error { - mqBody, err := json.Marshal(app) +func (a *AppAction) ImportApp(importApp *model.ImportAppStruct) error { + mqBody, err := json.Marshal(importApp) ts := &db.BuildTaskStruct{ TaskType: "import_app", diff --git a/api/model/model.go b/api/model/model.go index bf7df6e3d..2277562e9 100644 --- a/api/model/model.go +++ b/api/model/model.go @@ -1357,7 +1357,7 @@ type ExportAppStruct struct { EventID string `json:"event_id"` GroupKey string `json:"group_key"` // TODO 考虑去掉 Version string `json:"version"` // TODO 考虑去掉 - Format string `json:"format"` // only rainbond-app/docker-compose + Format string `json:"format"` // only rainbond-app/docker-compose GroupMetadata string `json:"group_metadata"` } } @@ -1380,7 +1380,7 @@ type MQBody struct { SourceDir string `json:"source_dir"` } -func NewAppStatusFrom(app *ExportAppStruct) *dbmodel.AppStatus { +func NewAppStatusFromExport(app *ExportAppStruct) *dbmodel.AppStatus { fields := strings.Split(app.SourceDir, "/") tarName := fields[len(fields)-1] tarFileHref := fmt.Sprintf("/v2/app/download/%s/%s.tar", app.Body.Format, tarName) @@ -1392,3 +1392,35 @@ func NewAppStatusFrom(app *ExportAppStruct) *dbmodel.AppStatus { TarFileHref: tarFileHref, } } + +type ImportAppStruct struct { + EventID string `json:"event_id"` + SourceDir string `json:"source_dir"` + Format string `json:"format"` + ServiceImage ServiceImage `json:"service_image"` + ServiceSlug ServiceSlug `json:"service_slug"` +} + +type ServiceImage struct { + HubUrl string `json:"hub_url"` + HubUser string `json:"hub_user"` + HubPassword string `json:"hub_password"` + NameSpace string `json:"namespace"` +} + +type ServiceSlug struct { + FtpHost string `json:"ftp_host"` + FtpPort string `json:"ftp_port"` + FtpUsername string `json:"ftp_username"` + FtpPassword string `json:"ftp_password"` + NameSpace string `json:"namespace"` +} + +func NewAppStatusFromImport(app *ImportAppStruct) *dbmodel.AppStatus { + return &dbmodel.AppStatus{ + EventID: app.EventID, + Format: app.Format, + SourceDir: app.SourceDir, + Status: "importing", + } +} diff --git a/builder/exector/import_app.go b/builder/exector/import_app.go index e0f52229b..d711f42f9 100644 --- a/builder/exector/import_app.go +++ b/builder/exector/import_app.go @@ -19,20 +19,22 @@ package exector import ( + "errors" "fmt" "github.com/Sirupsen/logrus" + "github.com/bitly/go-simplejson" + "github.com/docker/engine-api/client" + "github.com/goodrain/rainbond/builder/sources" + "github.com/goodrain/rainbond/db" + dbmodel "github.com/goodrain/rainbond/db/model" + "github.com/goodrain/rainbond/event" + "github.com/tidwall/gjson" "io/ioutil" "os/exec" - "strings" "path/filepath" - "github.com/goodrain/rainbond/builder/sources" - "github.com/tidwall/gjson" - "github.com/goodrain/rainbond/event" - "github.com/docker/engine-api/client" + "strings" "time" - "github.com/goodrain/rainbond/db" - "errors" - "github.com/goodrain/rainbond/db/model" + "github.com/goodrain/rainbond/api/model" ) func init() { @@ -44,6 +46,8 @@ type ImportApp struct { EventID string `json:"event_id"` Format string `json:"format"` SourceDir string `json:"source_dir"` + ServiceImage model.ServiceImage + ServiceSlug model.ServiceSlug Logger event.Logger DockerClient *client.Client } @@ -57,10 +61,15 @@ func NewImportApp(in []byte) TaskWorker { } eventID := gjson.GetBytes(in, "event_id").String() + serviceImage := gjson.GetBytes(in, "service_image").Value().(model.ServiceImage) + serviceSlug := gjson.GetBytes(in, "service_image").Value().(model.ServiceSlug) logger := event.GetManager().GetLogger(eventID) + return &ImportApp{ Format: gjson.GetBytes(in, "format").String(), SourceDir: gjson.GetBytes(in, "source_dir").String(), + ServiceImage: serviceImage, + ServiceSlug: serviceSlug, Logger: logger, EventID: eventID, DockerClient: dockerClient, @@ -103,12 +112,42 @@ func (i *ImportApp) importApp() error { return err } - // 上传镜像和源码包到仓库中 - if err := i.loadApps(); err != nil { + // 如果目录下有多个子目录，则认为每个子目录是一个应用组，循环导入之 + dirs, err := ioutil.ReadDir(i.SourceDir) + if err != nil { + logrus.Error("Failed to read dir for import app: ", i.SourceDir) return err } - // 更橷应用状态 + oldSourceDir := i.SourceDir + var errMsg string + for _, dir := range dirs { + if dir.IsDir() { + continue + } + + i.SourceDir = filepath.Join(i.SourceDir, dir.Name()) + + // 修改json元数据中的镜像和源码包仓库地址为指定地址 + if err := i.replaceRepo(); err != nil { + logrus.Errorf("Failed to change repo address in metadata json for %s: %s", i.SourceDir, err) + errMsg = fmt.Sprintf("%s; %s", errMsg, err.Error()) + continue + } + + // 上传镜像和源码包到仓库中 + if err := i.loadApps(); err != nil { + logrus.Errorf("Failed to load apps for %s: %s", i.SourceDir, err) + errMsg = fmt.Sprintf("%s; %s", errMsg, err.Error()) + continue + } + } + i.SourceDir = oldSourceDir + if errMsg != "" { + return errors.New(errMsg) + } + + // 更新应用状态 if err := i.updateStatus("success"); err != nil { return err } @@ -116,8 +155,9 @@ func (i *ImportApp) importApp() error { return nil } +// i.SourceDir = "/grdata/app/import/n7brv4/web-app" func (i *ImportApp) unzip() error { - cmd := fmt.Sprintf("cd %s && rm -rf %s && tar -xf %s.tar", filepath.Dir(i.SourceDir), i.SourceDir, i.SourceDir) + cmd := fmt.Sprintf("cd %s && tar -xf *.tar", i.SourceDir) err := exec.Command("sh", "-c", cmd).Run() if err != nil { logrus.Error("Failed to unzip for import app: ", i.SourceDir, ".tar") @@ -128,6 +168,45 @@ func (i *ImportApp) unzip() error { return err } +// 替换元数据中的镜像和源码包的仓库地址 +func (i *ImportApp) replaceRepo() error { + metaFile := fmt.Sprintf("%s/metadata.json", i.SourceDir) + logrus.Debug("Change image and slug repo address in: ", metaFile) + + data, err := ioutil.ReadFile(metaFile) + logrus.Debug("old json: ", string(data)) + meta, err := simplejson.NewJson(data) + if err != nil { + return err + } + + apps, err := meta.Get("apps").Array() + if err != nil { + return err + } + + for index := range apps { + app := meta.Get("apps").GetIndex(index) + if _, ok := app.CheckGet("service_image"); ok { + app.Set("service_image", i.ServiceImage) + } + + if _, ok := app.CheckGet("service_slug"); ok { + app.Set("service_slug", i.ServiceSlug) + } + apps[index] = app + } + + meta.Set("apps", apps) + data, err = meta.Bytes() + if err != nil { + return err + } + logrus.Debug("new json: ", string(data)) + + return ioutil.WriteFile(metaFile, data, 0644) +} + //parseApps get apps array from metadata.json func (i *ImportApp) parseApps() ([]gjson.Result, error) { i.Logger.Info("解析应用信息", map[string]string{"step": "export-app", "status": "success"}) @@ -244,7 +323,7 @@ func (i *ImportApp) updateStatus(status string) error { } // 在数据库中更新该应用的状态信息 - app := res.(*model.AppStatus) + app := res.(*dbmodel.AppStatus) app.Status = status app.Metadata = string(data)