mirror of
https://gitee.com/rainbond/Rainbond.git
synced 2024-12-05 05:07:38 +08:00
333 lines
13 KiB
Go
333 lines
13 KiB
Go
// RAINBOND, Application Management Platform
|
|
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
|
|
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version. For any non-GPL usage of Rainbond,
|
|
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
|
|
// must be obtained first.
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package exector
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/goodrain/rainbond/util"
|
|
|
|
"github.com/goodrain/rainbond/builder/sources"
|
|
"github.com/goodrain/rainbond/builder/sources/registry"
|
|
"github.com/goodrain/rainbond/db"
|
|
|
|
"github.com/Sirupsen/logrus"
|
|
"github.com/docker/docker/client"
|
|
dbmodel "github.com/goodrain/rainbond/db/model"
|
|
"github.com/goodrain/rainbond/event"
|
|
"github.com/pquerna/ffjson/ffjson"
|
|
"github.com/tidwall/gjson"
|
|
)
|
|
|
|
//BackupAPPNew backup group app new version
|
|
type BackupAPPNew struct {
|
|
GroupID string `json:"group_id" `
|
|
ServiceIDs []string `json:"service_ids" `
|
|
//full-online,full-offline
|
|
Mode string `json:"mode"`
|
|
Version string `json:"version"`
|
|
EventID string
|
|
SlugInfo struct {
|
|
Namespace string `json:"namespace"`
|
|
FTPHost string `json:"ftp_host"`
|
|
FTPPort string `json:"ftp_port"`
|
|
FTPUser string `json:"ftp_username"`
|
|
FTPPassword string `json:"ftp_password"`
|
|
} `json:"slug_info"`
|
|
ImageInfo struct {
|
|
HubURL string `json:"hub_url"`
|
|
HubUser string `json:"hub_user"`
|
|
HubPassword string `json:"hub_password"`
|
|
Namespace string `json:"namespace"`
|
|
IsTrust bool `json:"is_trust,omitempty"`
|
|
} `json:"image_info,omitempty"`
|
|
SourceDir string `json:"source_dir"`
|
|
SourceType string `json:"source_type"`
|
|
BackupID string `json:"backup_id"`
|
|
BackupSize int64
|
|
Logger event.Logger
|
|
DockerClient *client.Client
|
|
}
|
|
|
|
func init() {
|
|
RegisterWorker("backup_apps_new", BackupAPPNewCreater)
|
|
}
|
|
|
|
//BackupAPPNewCreater create
|
|
func BackupAPPNewCreater(in []byte, m *exectorManager) (TaskWorker, error) {
|
|
eventID := gjson.GetBytes(in, "event_id").String()
|
|
logger := event.GetManager().GetLogger(eventID)
|
|
backupNew := &BackupAPPNew{
|
|
Logger: logger,
|
|
EventID: eventID,
|
|
DockerClient: m.DockerClient,
|
|
}
|
|
if err := ffjson.Unmarshal(in, &backupNew); err != nil {
|
|
return nil, err
|
|
}
|
|
return backupNew, nil
|
|
}
|
|
|
|
//RegionServiceSnapshot RegionServiceSnapshot
|
|
type RegionServiceSnapshot struct {
|
|
ServiceID string
|
|
Service *dbmodel.TenantServices
|
|
ServiceProbe []*dbmodel.TenantServiceProbe
|
|
LBMappingPort []*dbmodel.TenantServiceLBMappingPort
|
|
ServiceEnv []*dbmodel.TenantServiceEnvVar
|
|
ServiceLabel []*dbmodel.TenantServiceLable
|
|
ServiceMntRelation []*dbmodel.TenantServiceMountRelation
|
|
PluginRelation []*dbmodel.TenantServicePluginRelation
|
|
ServiceRelation []*dbmodel.TenantServiceRelation
|
|
ServiceStatus string
|
|
ServiceVolume []*dbmodel.TenantServiceVolume
|
|
ServicePort []*dbmodel.TenantServicesPort
|
|
Versions []*dbmodel.VersionInfo
|
|
}
|
|
|
|
//Run Run
|
|
func (b *BackupAPPNew) Run(timeout time.Duration) error {
|
|
//read region group app metadata
|
|
metadata, err := ioutil.ReadFile(fmt.Sprintf("%s/region_apps_metadata.json", b.SourceDir))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var appSnapshots []*RegionServiceSnapshot
|
|
if err := ffjson.Unmarshal(metadata, &appSnapshots); err != nil {
|
|
return err
|
|
}
|
|
for _, app := range appSnapshots {
|
|
//backup app image or code slug file
|
|
b.Logger.Info(fmt.Sprintf("Start backup Application(%s) runtime", app.Service.ServiceAlias), map[string]string{"step": "backup_builder", "status": "starting"})
|
|
haveAtLastOneVersion := false
|
|
for _, version := range app.Versions {
|
|
if version.DeliveredType == "slug" && version.FinalStatus == "success" {
|
|
if ok, _ := b.checkVersionExist(version); !ok {
|
|
version.FinalStatus = "lost"
|
|
continue
|
|
}
|
|
if err := b.uploadSlug(app, version); err != nil {
|
|
logrus.Errorf("upload app slug file error.%s", err.Error())
|
|
return err
|
|
}
|
|
haveAtLastOneVersion = true
|
|
}
|
|
if version.DeliveredType == "image" && version.FinalStatus == "success" {
|
|
if ok, _ := b.checkVersionExist(version); !ok {
|
|
version.FinalStatus = "lost"
|
|
continue
|
|
}
|
|
if err := b.uploadImage(app, version); err != nil {
|
|
logrus.Errorf("upload app image error.%s", err.Error())
|
|
return err
|
|
}
|
|
haveAtLastOneVersion = true
|
|
}
|
|
}
|
|
if !haveAtLastOneVersion {
|
|
b.Logger.Error(fmt.Sprintf("Application(%s) Backup build version failure.", app.Service.ServiceAlias), map[string]string{"step": "backup_builder", "status": "success"})
|
|
return fmt.Errorf("Application(%s) Backup build version failure", app.Service.ServiceAlias)
|
|
}
|
|
b.Logger.Info(fmt.Sprintf("Complete backup application (%s) runtime", app.Service.ServiceAlias), map[string]string{"step": "backup_builder", "status": "success"})
|
|
b.Logger.Info(fmt.Sprintf("Start backup application(%s) persistent data", app.Service.ServiceAlias), map[string]string{"step": "backup_builder", "status": "starting"})
|
|
//backup app data
|
|
for _, volume := range app.ServiceVolume {
|
|
if volume.HostPath != "" && !util.DirIsEmpty(volume.HostPath) {
|
|
dstDir := fmt.Sprintf("%s/data_%s/%s.zip", b.SourceDir, app.ServiceID, strings.Replace(volume.VolumeName, "/", "", -1))
|
|
if err := util.Zip(volume.HostPath, dstDir); err != nil {
|
|
logrus.Errorf("backup service(%s) volume(%s) data error.%s", app.ServiceID, volume.VolumeName, err.Error())
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
b.Logger.Info(fmt.Sprintf("Complete backup application(%s) persistent data", app.Service.ServiceAlias), map[string]string{"step": "backup_builder", "status": "success"})
|
|
//TODO:backup relation volume data?
|
|
}
|
|
if strings.HasSuffix(b.SourceDir, "/") {
|
|
b.SourceDir = b.SourceDir[:len(b.SourceDir)-2]
|
|
}
|
|
if err := util.Zip(b.SourceDir, fmt.Sprintf("%s.zip", b.SourceDir)); err != nil {
|
|
b.Logger.Info(fmt.Sprintf("Compressed backup metadata failed"), map[string]string{"step": "backup_builder", "status": "starting"})
|
|
return err
|
|
}
|
|
b.BackupSize += util.GetFileSize(fmt.Sprintf("%s.zip", b.SourceDir))
|
|
os.RemoveAll(b.SourceDir)
|
|
b.SourceDir = fmt.Sprintf("%s.zip", b.SourceDir)
|
|
//upload app backup data to online server(sftp) if mode is full-online
|
|
if b.Mode == "full-online" && b.SlugInfo.FTPHost != "" && b.SlugInfo.FTPPort != "" {
|
|
b.Logger.Info(fmt.Sprintf("Start uploading backup metadata to the cloud"), map[string]string{"step": "backup_builder", "status": "starting"})
|
|
sFTPClient, err := sources.NewSFTPClient(b.SlugInfo.FTPUser, b.SlugInfo.FTPPassword, b.SlugInfo.FTPHost, b.SlugInfo.FTPPort)
|
|
if err != nil {
|
|
b.Logger.Error(util.Translation("create ftp client error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
return err
|
|
}
|
|
defer sFTPClient.Close()
|
|
dstDir := fmt.Sprintf("%s/backup/%s_%s/metadata_data.zip", b.SlugInfo.Namespace, b.GroupID, b.Version)
|
|
if err := sFTPClient.PushFile(b.SourceDir, dstDir, b.Logger); err != nil {
|
|
b.Logger.Error(util.Translation("push slug file to sftp server error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
logrus.Errorf("push slug file error when backup app , %s", err.Error())
|
|
return err
|
|
}
|
|
//Statistical backup size
|
|
os.Remove(b.SourceDir)
|
|
b.SourceDir = dstDir
|
|
b.SourceType = "sftp"
|
|
}
|
|
if err := b.updateBackupStatu("success"); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
func (b *BackupAPPNew) checkVersionExist(version *dbmodel.VersionInfo) (bool, error) {
|
|
if version.DeliveredType == "image" {
|
|
imageInfo := sources.ImageNameHandle(version.DeliveredPath)
|
|
reg, err := registry.NewInsecure(imageInfo.Host, "", "")
|
|
if err != nil {
|
|
logrus.Errorf("new registry client error %s", err.Error())
|
|
return false, err
|
|
}
|
|
_, err = reg.Manifest(imageInfo.Name, imageInfo.Tag)
|
|
if err != nil {
|
|
logrus.Errorf("get image %s manifest info failure, it could be not exist", version.DeliveredPath)
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
if version.DeliveredType == "slug" {
|
|
islugfile, err := os.Stat(version.DeliveredPath)
|
|
if os.IsNotExist(err) {
|
|
return false, nil
|
|
} else if err != nil {
|
|
return false, err
|
|
}
|
|
if islugfile.IsDir() {
|
|
return false, nil
|
|
}
|
|
return true, nil
|
|
}
|
|
return false, fmt.Errorf("delivered type is invalid")
|
|
}
|
|
func (b *BackupAPPNew) uploadSlug(app *RegionServiceSnapshot, version *dbmodel.VersionInfo) error {
|
|
if b.Mode == "full-online" && b.SlugInfo.FTPHost != "" && b.SlugInfo.FTPPort != "" {
|
|
sFTPClient, err := sources.NewSFTPClient(b.SlugInfo.FTPUser, b.SlugInfo.FTPPassword, b.SlugInfo.FTPHost, b.SlugInfo.FTPPort)
|
|
if err != nil {
|
|
b.Logger.Error(util.Translation("create ftp client error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
return err
|
|
}
|
|
defer sFTPClient.Close()
|
|
dstDir := fmt.Sprintf("%s/backup/%s_%s/app_%s/%s.tgz", b.SlugInfo.Namespace, b.GroupID, b.Version, app.ServiceID, version.BuildVersion)
|
|
if err := sFTPClient.PushFile(version.DeliveredPath, dstDir, b.Logger); err != nil {
|
|
b.Logger.Error(util.Translation("push slug file to sftp server error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
logrus.Errorf("push slug file error when backup app , %s", err.Error())
|
|
return err
|
|
}
|
|
//Statistical backup size
|
|
b.BackupSize += util.GetFileSize(version.DeliveredPath)
|
|
} else {
|
|
dstDir := fmt.Sprintf("%s/app_%s/slug_%s.tgz", b.SourceDir, app.ServiceID, version.BuildVersion)
|
|
util.CheckAndCreateDir(filepath.Dir(dstDir))
|
|
if err := sources.CopyFileWithProgress(version.DeliveredPath, dstDir, b.Logger); err != nil {
|
|
b.Logger.Error(util.Translation("push slug file to local dir error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
logrus.Errorf("copy slug file error when backup app, %s", err.Error())
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *BackupAPPNew) uploadImage(app *RegionServiceSnapshot, version *dbmodel.VersionInfo) error {
|
|
if b.Mode == "full-online" && b.ImageInfo.HubURL != "" {
|
|
backupImage, err := version.CreateShareImage(b.ImageInfo.HubURL, b.ImageInfo.Namespace, fmt.Sprintf("%s_backup", b.Version))
|
|
if err != nil {
|
|
return fmt.Errorf("create backup image error %s", err)
|
|
}
|
|
|
|
info, err := sources.ImagePull(b.DockerClient, version.DeliveredPath, "", "", b.Logger, 10)
|
|
if err != nil {
|
|
return fmt.Errorf("pull image when backup error %s", err)
|
|
}
|
|
if err := sources.ImageTag(b.DockerClient, version.DeliveredPath, backupImage, b.Logger, 1); err != nil {
|
|
return fmt.Errorf("change image tag when backup error %s", err)
|
|
}
|
|
if b.ImageInfo.IsTrust {
|
|
if err := sources.TrustedImagePush(b.DockerClient, backupImage, b.ImageInfo.HubUser, b.ImageInfo.HubPassword, b.Logger, 10); err != nil {
|
|
b.Logger.Error(util.Translation("save image to hub error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
return fmt.Errorf("backup image push error %s", err)
|
|
}
|
|
} else {
|
|
if err := sources.ImagePush(b.DockerClient, backupImage, b.ImageInfo.HubUser, b.ImageInfo.HubPassword, b.Logger, 10); err != nil {
|
|
b.Logger.Error(util.Translation("save image to hub error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
return fmt.Errorf("backup image push error %s", err)
|
|
}
|
|
}
|
|
b.BackupSize += info.Size
|
|
} else {
|
|
dstDir := fmt.Sprintf("%s/app_%s/image_%s.tar", b.SourceDir, app.ServiceID, version.BuildVersion)
|
|
util.CheckAndCreateDir(filepath.Dir(dstDir))
|
|
if err := sources.ImageSave(b.DockerClient, version.DeliveredPath, dstDir, b.Logger); err != nil {
|
|
b.Logger.Error(util.Translation("save image to local dir error"), map[string]string{"step": "backup_builder", "status": "failure"})
|
|
logrus.Errorf("save image(%s) to local dir error when backup app, %s", version.DeliveredPath, err.Error())
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
//Stop stop
|
|
func (b *BackupAPPNew) Stop() error {
|
|
return nil
|
|
}
|
|
|
|
//Name return worker name
|
|
func (b *BackupAPPNew) Name() string {
|
|
return "backup_apps_new"
|
|
}
|
|
|
|
//GetLogger GetLogger
|
|
func (b *BackupAPPNew) GetLogger() event.Logger {
|
|
return b.Logger
|
|
}
|
|
|
|
//ErrorCallBack if run error will callback
|
|
func (b *BackupAPPNew) ErrorCallBack(err error) {
|
|
if err != nil {
|
|
logrus.Errorf("backup group app failure %s", err)
|
|
b.Logger.Error(util.Translation("backup group app failure"), map[string]string{"step": "callback", "status": "failure"})
|
|
b.updateBackupStatu("failed")
|
|
}
|
|
}
|
|
|
|
func (b *BackupAPPNew) updateBackupStatu(status string) error {
|
|
backupstatus, err := db.GetManager().AppBackupDao().GetAppBackup(b.BackupID)
|
|
if err != nil {
|
|
logrus.Errorf("update backup group app history failure %s", err)
|
|
return err
|
|
}
|
|
backupstatus.Status = status
|
|
backupstatus.SourceDir = b.SourceDir
|
|
backupstatus.SourceType = b.SourceType
|
|
backupstatus.BuckupSize = int(b.BackupSize)
|
|
return db.GetManager().AppBackupDao().UpdateModel(backupstatus)
|
|
}
|