Rainbond/builder/exector/import_app.go
2020-04-15 18:41:09 +08:00

580 lines
19 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package exector
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/Sirupsen/logrus"
simplejson "github.com/bitly/go-simplejson"
"github.com/docker/docker/client"
"github.com/goodrain/rainbond/api/model"
"github.com/goodrain/rainbond/builder"
"github.com/goodrain/rainbond/builder/sources"
"github.com/goodrain/rainbond/db"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/util"
"github.com/tidwall/gjson"
)
func init() {
RegisterWorker("import_app", NewImportApp)
}
//ImportApp Export app to specified format(rainbond-app or dockercompose)
type ImportApp struct {
EventID string `json:"event_id"`
Format string `json:"format"`
SourceDir string `json:"source_dir"`
Apps []string `json:"apps"`
ServiceImage model.ServiceImage
ServiceSlug model.ServiceSlug
Logger event.Logger
DockerClient *client.Client
oldAPPPath map[string]string
oldPluginPath map[string]string
}
//NewImportApp create
func NewImportApp(in []byte, m *exectorManager) (TaskWorker, error) {
eventID := gjson.GetBytes(in, "event_id").String()
var serviceSlug model.ServiceSlug
if err := json.Unmarshal([]byte(gjson.GetBytes(in, "service_slug").String()), &serviceSlug); err != nil {
logrus.Error("Failed to unmarshal service_slug for import app: ", err)
return nil, err
}
apps := make([]string, 0, 10)
for _, r := range gjson.GetBytes(in, "apps").Array() {
apps = append(apps, r.String())
}
logger := event.GetManager().GetLogger(eventID)
return &ImportApp{
Format: gjson.GetBytes(in, "format").String(),
SourceDir: gjson.GetBytes(in, "source_dir").String(),
Apps: apps,
ServiceSlug: serviceSlug,
Logger: logger,
EventID: eventID,
DockerClient: m.DockerClient,
oldAPPPath: make(map[string]string),
oldPluginPath: make(map[string]string),
}, nil
}
//Stop stop
func (i *ImportApp) Stop() error {
return nil
}
//Name return worker name
func (i *ImportApp) Name() string {
return "export_app"
}
//GetLogger GetLogger
func (i *ImportApp) GetLogger() event.Logger {
return i.Logger
}
//ErrorCallBack if run error will callback
func (i *ImportApp) ErrorCallBack(err error) {
i.updateStatus("failed", "")
}
//Run Run
func (i *ImportApp) Run(timeout time.Duration) error {
if i.Format == "rainbond-app" {
err := i.importApp()
if err != nil {
return err
}
return nil
}
return errors.New("Unsupported the format: " + i.Format)
}
// importApp import app
// support batch import
func (i *ImportApp) importApp() error {
oldSourceDir := i.SourceDir
var datas = "["
tmpDir := oldSourceDir + "/TmpUnzipDir"
for _, app := range i.Apps {
if err := i.updateStatusForApp(app, "importing"); err != nil {
logrus.Errorf("Failed to update status to importing for app %s: %v", app, err)
}
appFile := filepath.Join(oldSourceDir, app)
os.MkdirAll(tmpDir, 0755)
err := util.Unzip(appFile, tmpDir)
if err != nil {
logrus.Errorf("Failed to unzip app file %s : %s", appFile, err.Error())
i.updateStatusForApp(app, "failed")
continue
}
files, _ := ioutil.ReadDir(tmpDir)
if len(files) < 1 {
logrus.Errorf("Failed to read files in tmp dir %s: %v", appFile, err)
continue
}
i.SourceDir = fmt.Sprintf("%s/%s", oldSourceDir, files[0].Name())
if _, err := os.Stat(i.SourceDir); err == nil {
os.RemoveAll(i.SourceDir)
}
err = os.Rename(fmt.Sprintf("%s/%s", tmpDir, files[0].Name()), i.SourceDir)
if err != nil {
logrus.Errorf("Failed to mv source dir to %s: %v", i.SourceDir, err)
continue
}
os.RemoveAll(tmpDir)
// push image and slug file
// 修改json元数据中的镜像和源码包仓库地址为指定地址
metaFile := fmt.Sprintf("%s/metadata.json", i.SourceDir)
logrus.Debug("Change image and slug repo address in: ", metaFile)
data, err := ioutil.ReadFile(metaFile)
meta, err := simplejson.NewJson(data)
if err != nil {
logrus.Errorf("Failed to new json for load app %s: %v", appFile, err)
i.updateStatusForApp(app, "failed")
continue
}
apps, err := meta.Get("apps").Array()
if err != nil {
logrus.Errorf("Failed to get apps from meta for load app %s: %v", appFile, err)
i.updateStatusForApp(app, "failed")
continue
}
for index := range apps {
app := meta.Get("apps").GetIndex(index)
if _, ok := app.CheckGet("service_slug"); ok {
app.Set("service_slug", i.ServiceSlug)
}
getAppImage := func() string {
oldname, _ := app.Get("share_image").String()
oldImageName := sources.ImageNameWithNamespaceHandle(oldname)
hubURL, err := app.Get("service_image").Get("hub_url").String()
if err != nil {
logrus.Warn("app service image hubURL get failed: %s, use goodrain.me", err.Error())
hubURL = builder.REGISTRYDOMAIN
}
image := fmt.Sprintf("%s/%s:%s", hubURL, oldImageName.Name, oldImageName.Tag)
namespace, _ := app.Get("service_image").Get("namespace").String()
if namespace != "" {
image = fmt.Sprintf("%s/%s/%s:%s", hubURL, namespace, oldImageName.Name, oldImageName.Tag)
}
return image
}
getAppSlugPath := func() string {
shareSlugPath, _ := app.Get("share_slug_path").String()
if strings.HasPrefix(shareSlugPath, "/grdata/build/tenant/") {
shareSlugPath = strings.Replace(shareSlugPath, "/grdata/build/tenant/", "", 1)
}
if i.ServiceSlug.FtpHost == "" {
shareSlugPath = fmt.Sprintf("/grdata/build/tenant/%s", shareSlugPath)
} else {
info := strings.Split(shareSlugPath, "/")
shareSlugPath = fmt.Sprintf("%s/%s", i.ServiceSlug.NameSpace, strings.Join(info[1:], "/"))
}
return strings.Replace(shareSlugPath, "//", "/", -1)
}
if oldImage, ok := app.CheckGet("share_image"); ok {
appKey, _ := app.Get("service_key").String()
i.oldAPPPath[appKey], _ = oldImage.String()
app.Set("share_image", getAppImage())
}
if oldslug, ok := app.CheckGet("share_slug_path"); ok {
appKey, _ := app.Get("service_key").String()
i.oldAPPPath[appKey], _ = oldslug.String()
app.Set("share_slug_path", getAppSlugPath())
}
apps[index] = app
}
meta.Set("apps", apps)
data, err = meta.MarshalJSON()
if err != nil {
logrus.Errorf("Failed to marshal json for app %s: %v", appFile, err)
i.updateStatusForApp(app, "failed")
continue
}
err = ioutil.WriteFile(metaFile, data, 0644)
if err != nil {
logrus.Errorf("Failed to write metadata load app %s: %v", appFile, err)
i.updateStatusForApp(app, "failed")
continue
}
// load all service version attachment in app
if err := i.loadApps(); err != nil {
logrus.Errorf("Failed to load app %s: %v", appFile, err)
i.updateStatusForApp(app, "failed")
continue
}
// load all plugins
if err := i.importPlugins(); err != nil {
logrus.Errorf("Failed to load app plugin %s: %v", appFile, err)
i.updateStatusForApp(app, "failed")
continue
}
if err := i.updateStatusForApp(app, "success"); err != nil {
logrus.Errorf("Failed to update status to success for app %s: %v", app, err)
continue
}
if datas == "[" {
datas += string(data)
} else {
datas += ", " + string(data)
}
os.Rename(appFile, appFile+".success")
logrus.Debug("Successful import app: ", appFile)
}
datas += "]"
i.SourceDir = oldSourceDir
metadatasFile := fmt.Sprintf("%s/metadatas.json", i.SourceDir)
if err := ioutil.WriteFile(metadatasFile, []byte(datas), 0644); err != nil {
logrus.Errorf("Failed to load apps %s: %v", i.SourceDir, err)
return err
}
// 更新应用状态
if err := i.updateStatus("success", datas); err != nil {
logrus.Errorf("Failed to load apps %s: %v", i.SourceDir, err)
return err
}
return nil
}
//parseApps get apps array from metadata.json
func (i *ImportApp) parseApps() ([]gjson.Result, error) {
i.Logger.Info("解析应用信息", map[string]string{"step": "export-app", "status": "success"})
data, err := ioutil.ReadFile(fmt.Sprintf("%s/metadata.json", i.SourceDir))
if err != nil {
i.Logger.Error("导出应用失败,没有找到应用信息", map[string]string{"step": "read-metadata", "status": "failure"})
logrus.Error("Failed to read metadata file: ", err)
return nil, err
}
arr := gjson.GetBytes(data, "apps").Array()
if len(arr) < 1 {
i.Logger.Error("解析应用列表信息失败", map[string]string{"step": "parse-apps", "status": "failure"})
err := errors.New("not found apps in the metadata")
logrus.Error("Failed to get apps from json: ", err)
return nil, err
}
logrus.Debug("Successful parse apps array from metadata, count: ", len(arr))
return arr, nil
}
func (i *ImportApp) importPlugins() error {
i.Logger.Info("解析插件信息", map[string]string{"step": "import-plugins", "status": "success"})
data, err := ioutil.ReadFile(fmt.Sprintf("%s/metadata.json", i.SourceDir))
if err != nil {
i.Logger.Error("导出插件失败,没有找到应用信息", map[string]string{"step": "read-metadata", "status": "failure"})
logrus.Error("Failed to read metadata file: ", err)
return err
}
// 先修改json数据中的插件镜像服务器地址为新环境中的服务器地址
meta, err := simplejson.NewJson(data)
if err != nil {
logrus.Errorf("Failed to new json for load app %s: %v", i.SourceDir, err)
return err
}
oldPlugins, err := meta.Get("plugins").Array()
if err != nil {
logrus.Errorf("Failed to get plugins from meta for load app %s: %v", i.SourceDir, err)
return nil
}
for index := range oldPlugins {
plugin := meta.Get("plugins").GetIndex(index)
getImageImage := func() string {
oldname, _ := plugin.Get("share_image").String()
oldImageName := sources.ImageNameWithNamespaceHandle(oldname)
var image string
hubURL, err := plugin.Get("plugin_image").Get("hub_url").String()
if err != nil {
logrus.Warnf("plugin get hub url failed: %s, use goodrain.me", err.Error())
hubURL = builder.REGISTRYDOMAIN
}
image = fmt.Sprintf("%s/%s:%s", hubURL, oldImageName.Name, oldImageName.Tag)
namespace, _ := plugin.Get("plugin_image").Get("namespace").String()
if namespace != "" {
image = fmt.Sprintf("%s/%s/%s:%s", hubURL, namespace, oldImageName.Name, oldImageName.Tag)
}
return image
}
if oldimage, ok := plugin.CheckGet("share_image"); ok {
pluginID, _ := plugin.Get("plugin_id").String()
i.oldPluginPath[pluginID], _ = oldimage.String()
plugin.Set("share_image", getImageImage())
} else {
logrus.Warnf("plugin do not found share_image, skip it")
}
oldPlugins[index] = plugin
}
meta.Set("plugins", oldPlugins)
data, err = meta.MarshalJSON()
if err != nil {
logrus.Errorf("Failed to marshal json for app %s: %v", i.SourceDir, err)
return err
}
// 修改完毕后写回文件中
err = ioutil.WriteFile(fmt.Sprintf("%s/metadata.json", i.SourceDir), data, 0644)
if err != nil {
logrus.Errorf("Failed to write metadata load app %s: %v", i.SourceDir, err)
return err
}
plugins := gjson.GetBytes(data, "plugins").Array()
for _, plugin := range plugins {
pluginName := plugin.Get("plugin_name").String()
pluginName = unicode2zh(pluginName)
pluginDir := fmt.Sprintf("%s/%s", i.SourceDir, pluginName)
files, err := ioutil.ReadDir(pluginDir)
if err != nil || len(files) < 1 {
logrus.Error("Failed to list in service directory: ", pluginDir)
continue
}
fileName := filepath.Join(pluginDir, files[0].Name())
logrus.Debug("Parse the source file for service: ", fileName)
// 将镜像加载到本地,并上传到仓库
if strings.HasSuffix(fileName, ".image.tar") {
// 加载到本地
if err := sources.ImageLoad(i.DockerClient, fileName, i.Logger); err != nil {
logrus.Error("Failed to load image for service: ", pluginName)
return err
}
// 上传到仓库
// get docker account user and pass
user, pass := builder.GetImageUserInfo(plugin.Get("plugin_image.hub_user").String(), plugin.Get("plugin_image.hub_password").String())
// 上传之前先要根据新的仓库地址修改镜像名
image := i.oldPluginPath[plugin.Get("plugin_id").String()]
imageName := sources.ImageNameWithNamespaceHandle(image)
saveImageName := fmt.Sprintf("%s/%s:%s", builder.REGISTRYDOMAIN, imageName.Name, imageName.Tag)
newImageName := plugin.Get("share_image").String()
if err := sources.ImageTag(i.DockerClient, saveImageName, newImageName, i.Logger, 2); err != nil {
return fmt.Errorf("change plugin image tag(%s => %s) error %s", saveImageName, newImageName, err.Error())
}
// 开始上传
if err := sources.ImagePush(i.DockerClient, saveImageName, user, pass, i.Logger, 15); err != nil {
return fmt.Errorf("push plugin image %s error %s", image, err.Error())
}
logrus.Debug("Successful load and push the plugin image ", image)
}
}
return nil
}
func (i *ImportApp) loadApps() error {
apps, err := i.parseApps()
if err != nil {
return err
}
for _, app := range apps {
// 获取该组件资源文件
serviceName := app.Get("service_cname").String()
serviceName = unicode2zh(serviceName)
serviceDir := fmt.Sprintf("%s/%s", i.SourceDir, serviceName)
files, err := ioutil.ReadDir(serviceDir)
if err != nil || len(files) < 1 {
logrus.Error("Failed to list in service directory: ", serviceDir)
continue
}
fileName := filepath.Join(serviceDir, files[0].Name())
logrus.Debug("Parse the source file for service: ", fileName)
// 判断该用应资源是什么类型
// 如果是镜像,则加载到本地,并上传到仓库
// 如果slug文件则上传到ftp服务器
if strings.HasSuffix(fileName, ".image.tar") {
// 加载到本地
if err := sources.ImageLoad(i.DockerClient, fileName, i.Logger); err != nil {
logrus.Error("Failed to load image for service: ", serviceName)
return err
}
// 上传到仓库
oldImage := i.oldAPPPath[app.Get("service_key").String()]
oldImageName := sources.ImageNameWithNamespaceHandle(oldImage)
// get docker account user and pass
user, pass := builder.GetImageUserInfo(app.Get("service_image.hub_user").String(), app.Get("service_image.hub_password").String())
// 上传之前先要根据新的仓库地址修改镜像名
image := app.Get("share_image").String()
if err := sources.ImageTag(i.DockerClient, fmt.Sprintf("%s/%s:%s", builder.REGISTRYDOMAIN, oldImageName.Name, oldImageName.Tag), image, i.Logger, 15); err != nil {
return fmt.Errorf("change image tag(%s => %s) error %s", fmt.Sprintf("%s/%s:%s", builder.REGISTRYDOMAIN, oldImageName.Name, oldImageName.Tag), image, err.Error())
}
// 开始上传
if err := sources.ImagePush(i.DockerClient, image, user, pass, i.Logger, 15); err != nil {
return fmt.Errorf("push image %s error %s", image, err.Error())
}
logrus.Debug("Successful load and push the image ", image)
} else if strings.HasSuffix(fileName, ".tgz") {
shareSlugPath := app.Get("share_slug_path").String()
ftpHost := app.Get("service_slug.ftp_host").String()
ftpPort := app.Get("service_slug.ftp_port").String()
ftpUsername := app.Get("service_slug.ftp_username").String()
ftpPassword := app.Get("service_slug.ftp_password").String()
// if sftp available
if ftpHost != "" && ftpPort != "" {
ftpClient, err := sources.NewSFTPClient(ftpUsername, ftpPassword, ftpHost, ftpPort)
if err != nil {
return err
}
err = ftpClient.PushFile(fileName, shareSlugPath, i.Logger)
ftpClient.Close()
if err != nil {
logrus.Errorf("Failed to upload slug file for group %s: %v", i.SourceDir, err)
return err
}
} else {
if err := util.CheckAndCreateDir(filepath.Dir(shareSlugPath)); err != nil {
logrus.Errorf("Failed create slug file directory %s error %s ", filepath.Dir(shareSlugPath), err.Error())
return err
}
err := util.CopyFile(fileName, shareSlugPath)
if err != nil {
logrus.Error("Failed to copy slug file to local directory: ", shareSlugPath)
return err
}
}
logrus.Debug("Successful upload slug file: ", fileName)
}
}
logrus.Debug("Successful load apps for group: ", i.SourceDir)
return nil
}
func (i *ImportApp) updateStatus(status, data string) error {
logrus.Debug("Update app status in database to: ", status)
// 从数据库中获取该应用的状态信息
res, err := db.GetManager().AppDao().GetByEventId(i.EventID)
if err != nil {
err = fmt.Errorf("failed to get app %s from db: %s", i.EventID, err.Error())
logrus.Error(err)
return err
}
// 在数据库中更新该应用的状态信息
res.Status = status
if err := db.GetManager().AppDao().UpdateModel(res); err != nil {
err = fmt.Errorf("failed to update app %s: %s", i.EventID, err.Error())
logrus.Error(err)
return err
}
return nil
}
func (i *ImportApp) updateStatusForApp(app, status string) error {
logrus.Debugf("Update status in database for app %s to: %s", app, status)
// 从数据库中获取该应用的状态信息
res, err := db.GetManager().AppDao().GetByEventId(i.EventID)
if err != nil {
err = fmt.Errorf("Failed to get app %s from db: %s", i.EventID, err.Error())
logrus.Error(err)
return err
}
// 在数据库中更新该应用的状态信息
appsMap := str2map(res.Apps)
appsMap[app] = status
res.Apps = map2str(appsMap)
if err := db.GetManager().AppDao().UpdateModel(res); err != nil {
err = fmt.Errorf("Failed to update app %s: %s", i.EventID, err.Error())
logrus.Error(err)
return err
}
return nil
}
func str2map(str string) map[string]string {
result := make(map[string]string, 10)
for _, app := range strings.Split(str, ",") {
appMap := strings.Split(app, ":")
result[appMap[0]] = appMap[1]
}
return result
}
func map2str(m map[string]string) string {
var result string
for k, v := range m {
kv := k + ":" + v
if result == "" {
result += kv
} else {
result += "," + kv
}
}
return result
}
// 只保留"/"后面的部分,并去掉不合法字符,一般用于把导出的镜像文件还原为镜像名
func buildFromLinuxFileName(fileName string) string {
if fileName == "" {
return fileName
}
arr := strings.Split(fileName, "/")
if str := arr[len(arr)-1]; str == "" {
fileName = strings.Replace(fileName, "---", "/", -1)
} else {
fileName = str
}
fileName = strings.Replace(fileName, "--", ":", -1)
fileName = strings.TrimSpace(fileName)
return fileName
}