test/Rainbond
1
0
Fork 0
mirror of https://gitee.com/rainbond/Rainbond.git synced 2024-11-30 02:38:17 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[ADD] add restore backup feature code (#72)

Browse Source
This commit is contained in:
barnett 2018-05-25 18:24:17 +08:00
parent 2d453cba42
commit 43bd88546c
20 changed files with 707 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/controller/group.go
View File

@ -24,6 +24,7 @@ import (
"github.com/go-chi/chi"
"github.com/goodrain/rainbond/api/handler"
"github.com/goodrain/rainbond/api/handler/group"
"github.com/goodrain/rainbond/api/middleware"
httputil "github.com/goodrain/rainbond/util/http"
)
@ -66,6 +67,8 @@ func Restore(w http.ResponseWriter, r *http.Request) {
return
}
br.BackupID = chi.URLParam(r, "backup_id")
tenantID := r.Context().Value(middleware.ContextKey("tenant_id")).(string)
br.Body.TenantID = tenantID
bean, err := handler.GetAPPBackupHandler().RestoreBackup(br)
if err != nil {
err.Handle(r, w)

18
api/handler/group/group_backup.go
View File

@ -304,6 +304,12 @@ type BackupRestore struct {
IsTrust bool `json:"is_trust,omitempty"`
} `json:"image_info,omitempty"`
EventID string `json:"event_id"`
//need restore target tenant id
TenantID string `json:"tenant_id"`
//RestoreMode(cdct) current datacenter and current tenant
//RestoreMode(cdot) current datacenter and other tenant
//RestoreMode(od) other datacenter
RestoreMode string `json:"restore_mode"`
}
}
@ -315,14 +321,18 @@ func (h *BackupHandle) RestoreBackup(br BackupRestore) (*dbmodel.AppBackup, *uti
if Aerr != nil {
return nil, Aerr
}
if backup.Status != "success" || backup.SourceDir == "" || backup.SourceType == "" {
return nil, util.CreateAPIHandleErrorf(500, "backup can not be restore")
}
var dataMap = map[string]interface{}{
"slug_info": br.Body.SlugInfo,
"image_info": br.Body.ImageInfo,
"backup_data": backup,
"slug_info": br.Body.SlugInfo,
"image_info": br.Body.ImageInfo,
"backup_id": backup.BackupID,
"tenant_id": br.Body.TenantID,
}
data, err := ffjson.Marshal(dataMap)
if err != nil {
return nil, util.CreateAPIHandleError(500, fmt.Errorf("build task body data error,%s", err))
return nil, util.CreateAPIHandleErrorf(500, "build task body data error,%s", err)
}
//Initiate a data backup task.
task := &apidb.BuildTaskStruct{

11
api/util/handleError.go
View File

@ -19,11 +19,12 @@
package util
import (
httputil "github.com/goodrain/rainbond/util/http"
"fmt"
"net/http"
"strings"
httputil "github.com/goodrain/rainbond/util/http"
"github.com/jinzhu/gorm"
"github.com/Sirupsen/logrus"
@ -43,6 +44,14 @@ func CreateAPIHandleError(code int, err error) *APIHandleError {
}
}
//CreateAPIHandleErrorf create handle error
func CreateAPIHandleErrorf(code int, format string, args ...interface{}) *APIHandleError {
return &APIHandleError{
Code: code,
Err: fmt.Errorf(format, args...),
}
}
//CreateAPIHandleErrorFromDBError from db error create APIHandleError
func CreateAPIHandleErrorFromDBError(msg string, err error) *APIHandleError {
if err.Error() == gorm.ErrRecordNotFound.Error() {

18
builder/exector/build_from_image_run.go
View File

@ -26,7 +26,7 @@ import (
"github.com/goodrain/rainbond/event"
"github.com/tidwall/gjson"
//"github.com/docker/docker/api/types"
"github.com/docker/engine-api/types"
//"github.com/docker/docker/client"
"github.com/docker/engine-api/client"
@ -72,21 +72,7 @@ func NewImageBuildItem(in []byte) *ImageBuildItem {
//Run Run
func (i *ImageBuildItem) Run(timeout time.Duration) error {
var pullipo types.ImagePullOptions
if i.HubUser != "" && i.HubPassword != "" {
auth, err := sources.EncodeAuthToBase64(types.AuthConfig{Username: i.HubUser, Password: i.HubPassword})
if err != nil {
logrus.Errorf("make auth base63 push image error: %s", err.Error())
i.Logger.Error(fmt.Sprintf("生成获取镜像的Token失败"), map[string]string{"step": "builder-exector", "status": "failure"})
return err
}
pullipo = types.ImagePullOptions{
RegistryAuth: auth,
}
} else {
pullipo = types.ImagePullOptions{}
}
_, err := sources.ImagePull(i.DockerClient, i.Image, pullipo, i.Logger, 10)
_, err := sources.ImagePull(i.DockerClient, i.Image, i.HubUser, i.HubPassword, i.Logger, 10)
if err != nil {
logrus.Errorf("pull image %s error: %s", i.Image, err.Error())
i.Logger.Error(fmt.Sprintf("获取指定镜像: %s失败", i.Image), map[string]string{"step": "builder-exector", "status": "failure"})

3
builder/exector/exector.go
View File

@ -32,6 +32,7 @@ import (
dbmodel "github.com/goodrain/rainbond/db/model"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/mq/api/grpc/pb"
"github.com/goodrain/rainbond/util"
"github.com/tidwall/gjson"
)
@ -139,7 +140,7 @@ func (e *exectorManager) exec(workerName string, in []byte) error {
if r := recover(); r != nil {
fmt.Println(r)
debug.PrintStack()
worker.GetLogger().Error("后端服务开小差，请重试或联系客服", map[string]string{"step": "callback", "status": "failure"})
worker.GetLogger().Error(util.Translation("Please try again or contact customer service"), map[string]string{"step": "callback", "status": "failure"})
worker.ErrorCallBack(fmt.Errorf("%s", r))
}
}()

9
builder/exector/export_app.go
View File

@ -30,14 +30,13 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/engine-api/client"
"github.com/docker/engine-api/types"
"github.com/goodrain/rainbond/builder/sources"
"github.com/goodrain/rainbond/db"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/util"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"gopkg.in/yaml.v2"
"github.com/goodrain/rainbond/util"
)
//ExportApp Export app to specified format(rainbond-app or dockercompose)
@ -263,7 +262,7 @@ func (i *ExportApp) exportImage(app gjson.Result) error {
}
// docker pull image-name
_, err := sources.ImagePull(i.DockerClient, image, types.ImagePullOptions{}, i.Logger, 15)
_, err := sources.ImagePull(i.DockerClient, image, "", "", i.Logger, 15)
if err != nil {
i.Logger.Error(fmt.Sprintf("拉取镜像失败：%s", image),
map[string]string{"step": "pull-image", "status": "failure"})
@ -422,7 +421,7 @@ func (i *ExportApp) exportRunnerImage() error {
return nil
}
_, err = sources.ImagePull(i.DockerClient, image, types.ImagePullOptions{}, i.Logger, 5)
_, err = sources.ImagePull(i.DockerClient, image, "", "", i.Logger, 10)
if err != nil {
i.Logger.Error(fmt.Sprintf("拉取镜像失败：%s", image),
map[string]string{"step": "pull-image", "status": "failure"})
@ -596,7 +595,7 @@ func (i *ExportApp) ErrorCallBack(err error) {
}
func (i *ExportApp) zip() error {
err := util.Zip(i.SourceDir, i.SourceDir + ".tar")
err := util.Zip(i.SourceDir, i.SourceDir+".tar")
if err != nil {
i.Logger.Error("打包应用失败", map[string]string{"step": "export-app", "status": "failure"})
logrus.Errorf("Failed to create tar file for group %s: %v", i.SourceDir, err)

27
builder/exector/groupapp_backup.go
View File

@ -32,7 +32,6 @@ import (
"github.com/Sirupsen/logrus"
"github.com/docker/engine-api/client"
"github.com/docker/engine-api/types"
dbmodel "github.com/goodrain/rainbond/db/model"
"github.com/goodrain/rainbond/event"
"github.com/pquerna/ffjson/ffjson"
@ -124,21 +123,18 @@ func (b *BackupAPPNew) Run(timeout time.Duration) error {
for _, app := range appSnapshots {
//backup app image or code slug file
b.Logger.Info(fmt.Sprintf("开始备份应用(%s)运行环境", app.Service.ServiceAlias), map[string]string{"step": "backup_builder", "status": "starting"})
for i := range app.Versions {
if version := app.Versions[len(app.Versions)-1-i]; version != nil && version.BuildVersion == app.Service.DeployVersion {
if version.DeliveredType == "slug" && version.FinalStatus == "success" {
if err := b.uploadSlug(app, version); err != nil {
logrus.Errorf("upload app slug file error.%s", err.Error())
return err
}
for _, version := range app.Versions {
if version.DeliveredType == "slug" && version.FinalStatus == "success" {
if err := b.uploadSlug(app, version); err != nil {
logrus.Errorf("upload app slug file error.%s", err.Error())
return err
}
if version.DeliveredType == "image" && version.FinalStatus == "success" {
if err := b.uploadImage(app, version); err != nil {
logrus.Errorf("upload app image error.%s", err.Error())
return err
}
}
if version.DeliveredType == "image" && version.FinalStatus == "success" {
if err := b.uploadImage(app, version); err != nil {
logrus.Errorf("upload app image error.%s", err.Error())
return err
}
break
}
}
b.Logger.Info(fmt.Sprintf("完成备份应用(%s)运行环境", app.Service.ServiceAlias), map[string]string{"step": "backup_builder", "status": "success"})
@ -147,7 +143,6 @@ func (b *BackupAPPNew) Run(timeout time.Duration) error {
for _, volume := range app.ServiceVolume {
if volume.HostPath != "" && !util.DirIsEmpty(volume.HostPath) {
dstDir := fmt.Sprintf("%s/data_%s/%s.zip", b.SourceDir, app.ServiceID, strings.Replace(volume.VolumeName, "/", "", -1))
util.CheckAndCreateDir(filepath.Dir(dstDir))
if err := util.Zip(volume.HostPath, dstDir); err != nil {
logrus.Errorf("backup service(%s) volume(%s) data error.%s", app.ServiceID, volume.VolumeName, err.Error())
return err
@ -226,7 +221,7 @@ func (b *BackupAPPNew) uploadImage(app *RegionServiceSnapshot, version *dbmodel.
if err != nil {
return fmt.Errorf("create backup image error %s", err)
}
info, err := sources.ImagePull(b.DockerClient, version.DeliveredPath, types.ImagePullOptions{}, b.Logger, 10)
info, err := sources.ImagePull(b.DockerClient, version.DeliveredPath, "", "", b.Logger, 10)
if err != nil {
return fmt.Errorf("pull image when backup error %s", err)
}

498
builder/exector/groupapp_restore.go Normal file
View File

@ -0,0 +1,498 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package exector
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/goodrain/rainbond/builder/sources"
dbmodel "github.com/goodrain/rainbond/db/model"
"github.com/Sirupsen/logrus"
"github.com/docker/engine-api/client"
"github.com/goodrain/rainbond/db"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/util"
"github.com/pquerna/ffjson/ffjson"
"github.com/tidwall/gjson"
)
//BackupAPPRestore restrore the group app backup
type BackupAPPRestore struct {
//full-online,full-offline
EventID string
SlugInfo struct {
Namespace string `json:"namespace"`
FTPHost string `json:"ftp_host"`
FTPPort string `json:"ftp_port"`
FTPUser string `json:"ftp_username"`
FTPPassword string `json:"ftp_password"`
} `json:"slug_info"`
ImageInfo struct {
HubURL string `json:"hub_url"`
HubUser string `json:"hub_user"`
HubPassword string `json:"hub_password"`
Namespace string `json:"namespace"`
IsTrust bool `json:"is_trust,omitempty"`
} `json:"image_info,omitempty"`
BackupID string `json:"backup_id"`
TenantID string `json:"tenant_id"`
Logger event.Logger
//RestoreMode(cdct) current datacenter and current tenant
//RestoreMode(cdot) current datacenter and other tenant
//RestoreMode(od) other datacenter
RestoreMode string `json:"restore_mode"`
DockerClient *client.Client
cacheDir string
serviceChange map[string]*Info
}
//Info service cache info
type Info struct {
ServiceID string
ServiceAlias string
Status string
LBPorts map[int]int
}
func init() {
RegisterWorker("backup_apps_restore", BackupAPPRestoreCreater)
}
//BackupAPPRestoreCreater create
func BackupAPPRestoreCreater(in []byte) (TaskWorker, error) {
dockerClient, err := client.NewEnvClient()
if err != nil {
logrus.Error("Failed to create task for export app: ", err)
return nil, err
}
eventID := gjson.GetBytes(in, "event_id").String()
logger := event.GetManager().GetLogger(eventID)
backupRestore := &BackupAPPRestore{
Logger: logger,
EventID: eventID,
DockerClient: dockerClient,
serviceChange: make(map[string]*Info, 0),
}
if err := ffjson.Unmarshal(in, &backupRestore); err != nil {
return nil, err
}
return backupRestore, nil
}
//Run Run
func (b *BackupAPPRestore) Run(timeout time.Duration) error {
//download or copy backup data
backup, err := db.GetManager().AppBackupDao().GetAppBackup(b.BackupID)
if err != nil {
return err
}
if backup.Status != "success" || backup.SourceDir == "" || backup.SourceType == "" {
return fmt.Errorf("backup can not be restore")
}
cacheDir := fmt.Sprintf("/grdata/cache/tmp/%s/%s", b.BackupID, b.EventID)
if err := util.CheckAndCreateDir(cacheDir); err != nil {
return fmt.Errorf("create cache dir error %s", err.Error())
}
b.cacheDir = cacheDir
switch backup.SourceType {
case "sftp":
b.downloadFromFTP(backup.SourceDir)
default:
b.downloadFromLocal(backup.SourceDir)
}
//read metadata file
metadata, err := ioutil.ReadFile(fmt.Sprintf("%s/region_apps_metadata.json", b.cacheDir))
if err != nil {
return err
}
var appSnapshots []*RegionServiceSnapshot
if err := ffjson.Unmarshal(metadata, &appSnapshots); err != nil {
return err
}
b.Logger.Info("读取备份元数据完成", map[string]string{"step": "restore_builder", "status": "success"})
//modify the metadata
if err := b.modify(appSnapshots); err != nil {
return err
}
//restore metadata to db
if err := b.restoreMetadata(appSnapshots); err != nil {
return err
}
b.Logger.Info("恢复备份元数据完成", map[string]string{"step": "restore_builder", "status": "success"})
//If the following error occurs, delete the data from the database
//restore all app all builde version and data
if err := b.restoreVersionAndData(backup, appSnapshots); err != nil {
return err
}
//save result
return nil
}
func (b *BackupAPPRestore) restoreVersionAndData(backup *dbmodel.AppBackup, appSnapshots []*RegionServiceSnapshot) error {
for _, app := range appSnapshots {
//backup app image or code slug file
b.Logger.Info(fmt.Sprintf("开始恢复应用(%s)运行环境", app.Service.ServiceAlias), map[string]string{"step": "restore_builder", "status": "starting"})
for _, version := range app.Versions {
if version.DeliveredType == "slug" && version.FinalStatus == "success" {
if err := b.downloadSlug(backup, app, version); err != nil {
logrus.Errorf("download app slug file error.%s", err.Error())
return err
}
}
if version.DeliveredType == "image" && version.FinalStatus == "success" {
if err := b.downloadImage(backup, app, version); err != nil {
logrus.Errorf("download app image error.%s", err.Error())
return err
}
}
}
b.Logger.Info(fmt.Sprintf("完成恢复应用(%s)运行环境", app.Service.ServiceAlias), map[string]string{"step": "restore_builder", "status": "success"})
b.Logger.Info(fmt.Sprintf("开始恢复应用(%s)持久化数据", app.Service.ServiceAlias), map[string]string{"step": "restore_builder", "status": "starting"})
//restore app data
for _, volume := range app.ServiceVolume {
if volume.HostPath != "" {
dstDir := fmt.Sprintf("%s/data_%s/%s.zip", b.cacheDir, b.getOldServiceID(app.ServiceID), strings.Replace(volume.VolumeName, "/", "", -1))
tmpDir := fmt.Sprintf("/grdata/tmp/%s_%d", volume.ServiceID, volume.ID)
if err := util.Unzip(dstDir, tmpDir); err != nil {
logrus.Errorf("restore service(%s) volume(%s) data error.%s", app.ServiceID, volume.VolumeName, err.Error())
return err
}
err := os.Rename(filepath.Join(tmpDir, strings.Replace(volume.VolumeName, "/", "", -1)), volume.HostPath)
if err != nil {
return err
}
if err := os.Chmod(volume.HostPath, 0777); err != nil {
return err
}
}
}
b.Logger.Info(fmt.Sprintf("完成恢复应用(%s)持久化数据", app.Service.ServiceAlias), map[string]string{"step": "restore_builder", "status": "success"})
//TODO:relation relation volume data?
}
return nil
}
func (b *BackupAPPRestore) getOldServiceID(new string) string {
for k, v := range b.serviceChange {
if v.ServiceID == new {
return k
}
}
return ""
}
func (b *BackupAPPRestore) downloadSlug(backup *dbmodel.AppBackup, app *RegionServiceSnapshot, version *dbmodel.VersionInfo) error {
if backup.BackupMode == "full-online" && b.SlugInfo.FTPHost != "" && b.SlugInfo.FTPPort != "" {
sFTPClient, err := sources.NewSFTPClient(b.SlugInfo.FTPUser, b.SlugInfo.FTPPassword, b.SlugInfo.FTPHost, b.SlugInfo.FTPPort)
if err != nil {
b.Logger.Error(util.Translation("create ftp client error"), map[string]string{"step": "restore_builder", "status": "failure"})
return err
}
defer sFTPClient.Close()
dstDir := fmt.Sprintf("%s/backup/%s_%s/app_%s/%s.tgz", b.SlugInfo.Namespace, backup.GroupID, backup.Version, b.getOldServiceID(app.ServiceID), version.BuildVersion)
if err := sFTPClient.DownloadFile(dstDir, version.DeliveredPath, b.Logger); err != nil {
b.Logger.Error(util.Translation("down slug file from sftp server error"), map[string]string{"step": "restore_builder", "status": "failure"})
logrus.Errorf("down slug file error when backup app , %s", err.Error())
return err
}
} else {
dstDir := fmt.Sprintf("%s/app_%s/slug_%s.tgz", b.cacheDir, b.getOldServiceID(app.ServiceID), version.BuildVersion)
if err := sources.CopyFileWithProgress(dstDir, version.DeliveredPath, b.Logger); err != nil {
b.Logger.Error(util.Translation("down slug file from local dir error"), map[string]string{"step": "restore_builder", "status": "failure"})
logrus.Errorf("copy slug file error when backup app, %s", err.Error())
return err
}
}
return nil
}
func (b *BackupAPPRestore) downloadImage(backup *dbmodel.AppBackup, app *RegionServiceSnapshot, version *dbmodel.VersionInfo) error {
if backup.BackupMode == "full-online" && b.ImageInfo.HubURL != "" {
backupImage, err := app.Service.CreateShareImage(b.ImageInfo.HubURL, b.ImageInfo.Namespace, fmt.Sprintf("%s_backup", backup.Version))
if err != nil {
return fmt.Errorf("create backup image error %s", err)
}
if _, err := sources.ImagePull(b.DockerClient, backupImage, b.ImageInfo.HubUser, b.ImageInfo.HubPassword, b.Logger, 10); err != nil {
b.Logger.Error(util.Translation("pull image from hub error"), map[string]string{"step": "restore_builder", "status": "failure"})
return fmt.Errorf("restore backup image pull error %s", err)
}
if err := sources.ImageTag(b.DockerClient, backupImage, version.DeliveredPath, b.Logger, 1); err != nil {
return fmt.Errorf("change image tag when restore backup error %s", err)
}
err = sources.ImagePush(b.DockerClient, version.DeliveredPath, "", "", b.Logger, 10)
if err != nil {
return fmt.Errorf("push image to local when restore backup error %s", err)
}
} else {
dstDir := fmt.Sprintf("%s/app_%s/image_%s.tar", b.cacheDir, b.getOldServiceID(app.ServiceID), version.BuildVersion)
if err := sources.ImageLoad(b.DockerClient, dstDir, b.Logger); err != nil {
b.Logger.Error(util.Translation("load image to local hub error"), map[string]string{"step": "restore_builder", "status": "failure"})
logrus.Errorf("load image to local hub error when restore backup app, %s", err.Error())
return err
}
}
return nil
}
//if restore error, will clear
func (b *BackupAPPRestore) clear() {
//clear db
//clear version data
//clear cache data
}
func (b *BackupAPPRestore) modify(appSnapshots []*RegionServiceSnapshot) error {
for _, app := range appSnapshots {
oldServiceID := app.ServiceID
if b.RestoreMode == "cdot" || b.RestoreMode == "od" {
//change tenant
app.Service.TenantID = b.TenantID
for _, port := range app.ServicePort {
port.TenantID = b.TenantID
}
for _, relation := range app.ServiceRelation {
relation.TenantID = b.TenantID
}
for _, env := range app.ServiceEnv {
env.TenantID = b.TenantID
}
for _, smr := range app.ServiceMntRelation {
smr.TenantID = b.TenantID
}
}
//change service_id and service_alias
newServiceID := util.NewUUID()
newServiceAlias := "gr" + newServiceID[26:]
app.ServiceID = newServiceID
app.Service.ServiceID = newServiceID
app.Service.ServiceAlias = newServiceAlias
for _, a := range app.ServiceProbe {
a.ServiceID = newServiceID
}
for _, a := range app.LBMappingPort {
a.ServiceID = newServiceID
}
for _, a := range app.ServiceEnv {
a.ServiceID = newServiceID
}
for _, a := range app.ServiceLabel {
a.ServiceID = newServiceID
}
for _, a := range app.ServiceMntRelation {
a.ServiceID = newServiceID
}
for _, a := range app.PluginRelation {
a.ServiceID = newServiceID
}
for _, a := range app.ServiceRelation {
a.ServiceID = newServiceID
}
for _, a := range app.ServiceVolume {
a.ServiceID = newServiceID
}
for _, a := range app.ServicePort {
a.ServiceID = newServiceID
}
for _, a := range app.Versions {
a.ServiceID = newServiceID
}
b.serviceChange[oldServiceID] = &Info{
ServiceID: newServiceID,
ServiceAlias: newServiceAlias,
Status: app.ServiceStatus,
}
}
//modify relations
for _, app := range appSnapshots {
for _, a := range app.ServiceMntRelation {
a.DependServiceID = b.serviceChange[a.DependServiceID].ServiceID
}
for _, a := range app.ServiceRelation {
a.DependServiceID = b.serviceChange[a.DependServiceID].ServiceID
}
}
return nil
}
func (b *BackupAPPRestore) restoreMetadata(appSnapshots []*RegionServiceSnapshot) error {
tx := db.GetManager().Begin()
for _, app := range appSnapshots {
app.Service.ID = 0
if err := db.GetManager().TenantServiceDaoTransactions(tx).AddModel(app.Service); err != nil {
tx.Rollback()
return fmt.Errorf("create app when restore backup error. %s", err.Error())
}
for _, a := range app.ServiceProbe {
a.ID = 0
if err := db.GetManager().ServiceProbeDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app probe when restore backup error. %s", err.Error())
}
}
for _, a := range app.LBMappingPort {
a.ID = 0
if err := db.GetManager().TenantServiceLBMappingPortDaoTransactions(tx).AddModel(a); err != nil {
if strings.Contains(err.Error(), "is exist ") {
//modify the lb port
maport, err := db.GetManager().TenantServiceLBMappingPortDaoTransactions(tx).CreateTenantServiceLBMappingPort(a.ServiceID, a.ContainerPort)
if err != nil {
tx.Rollback()
return fmt.Errorf("create new app lb port when restore backup error. %s", err.Error())
}
if b.serviceChange[app.ServiceID].LBPorts == nil {
b.serviceChange[app.ServiceID].LBPorts = map[int]int{maport.ContainerPort: maport.Port}
} else {
b.serviceChange[app.ServiceID].LBPorts[maport.ContainerPort] = maport.Port
}
} else {
tx.Rollback()
return fmt.Errorf("create app probe when restore backup error. %s", err.Error())
}
}
}
for _, a := range app.ServiceEnv {
a.ID = 0
if err := db.GetManager().TenantServiceEnvVarDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app envs when restore backup error. %s", err.Error())
}
}
for _, a := range app.ServiceLabel {
a.ID = 0
if err := db.GetManager().TenantServiceLabelDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app labels when restore backup error. %s", err.Error())
}
}
for _, a := range app.ServiceMntRelation {
a.ID = 0
if err := db.GetManager().TenantServiceMountRelationDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app mount relation when restore backup error. %s", err.Error())
}
}
for _, a := range app.PluginRelation {
a.ID = 0
if err := db.GetManager().TenantServicePluginRelationDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app plugin when restore backup error. %s", err.Error())
}
}
for _, a := range app.ServiceRelation {
a.ID = 0
if err := db.GetManager().TenantServiceRelationDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app relation when restore backup error. %s", err.Error())
}
}
for _, a := range app.ServiceVolume {
a.ID = 0
if err := db.GetManager().TenantServiceVolumeDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app volume when restore backup error. %s", err.Error())
}
}
for _, a := range app.ServicePort {
a.ID = 0
if err := db.GetManager().TenantServicesPortDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app ports when restore backup error. %s", err.Error())
}
}
for _, a := range app.Versions {
a.ID = 0
if err := db.GetManager().VersionInfoDaoTransactions(tx).AddModel(a); err != nil {
tx.Rollback()
return fmt.Errorf("create app versions when restore backup error. %s", err.Error())
}
}
}
if err := tx.Commit().Error; err != nil {
tx.Rollback()
return err
}
return nil
}
func (b *BackupAPPRestore) downloadFromLocal(sourceDir string) error {
err := util.Unzip(sourceDir, b.cacheDir)
if err != nil {
b.Logger.Error(util.Translation("unzip metadata file error"), map[string]string{"step": "backup_builder", "status": "failure"})
logrus.Errorf("unzip file error when restore backup app , %s", err.Error())
return err
}
filename := filepath.Base(sourceDir)
infos := strings.Split(filename, ".")
b.cacheDir = filepath.Join(b.cacheDir, infos[0])
return nil
}
func (b *BackupAPPRestore) downloadFromFTP(sourceDir string) error {
sFTPClient, err := sources.NewSFTPClient(b.SlugInfo.FTPUser, b.SlugInfo.FTPPassword, b.SlugInfo.FTPHost, b.SlugInfo.FTPPort)
if err != nil {
b.Logger.Error(util.Translation("create ftp client error"), map[string]string{"step": "backup_builder", "status": "failure"})
return err
}
defer sFTPClient.Close()
dstDir := fmt.Sprintf("%s/%s", b.cacheDir, filepath.Base(sourceDir))
if err := sFTPClient.DownloadFile(sourceDir, dstDir, b.Logger); err != nil {
b.Logger.Error(util.Translation("down slug file from sftp server error"), map[string]string{"step": "backup_builder", "status": "failure"})
logrus.Errorf("down slug file error when restore backup app , %s", err.Error())
return err
}
err = util.Unzip(dstDir, b.cacheDir)
if err != nil {
b.Logger.Error(util.Translation("unzip metadata file error"), map[string]string{"step": "backup_builder", "status": "failure"})
logrus.Errorf("unzip file error when restore backup app , %s", err.Error())
return err
}
filename := filepath.Base(sourceDir)
infos := strings.Split(filename, ".")
b.cacheDir = filepath.Join(b.cacheDir, infos[0])
return nil
}
//Stop stop
func (b *BackupAPPRestore) Stop() error {
return nil
}
//Name return worker name
func (b *BackupAPPRestore) Name() string {
return "backup_apps_restore"
}
//GetLogger GetLogger
func (b *BackupAPPRestore) GetLogger() event.Logger {
return b.Logger
}
//ErrorCallBack if run error will callback
func (b *BackupAPPRestore) ErrorCallBack(err error) {
if err != nil {
logrus.Errorf("restore backup group app failure %s", err)
b.Logger.Error(util.Translation("restore backup group app failure"), map[string]string{"step": "callback", "status": "failure"})
b.clear()
}
}

90
builder/exector/groupapp_restore_test.go Normal file
View File

@ -0,0 +1,90 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package exector
import (
"fmt"
"testing"
"github.com/pquerna/ffjson/ffjson"
dbmodel "github.com/goodrain/rainbond/db/model"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/util"
)
func TestDownloadFromLocal(t *testing.T) {
var b = BackupAPPRestore{
BackupID: "test",
EventID: "test",
Logger: event.GetTestLogger(),
}
cacheDir := fmt.Sprintf("/tmp/%s/%s", b.BackupID, b.EventID)
if err := util.CheckAndCreateDir(cacheDir); err != nil {
t.Fatal("create cache dir error", err.Error())
}
b.cacheDir = cacheDir
if err := b.downloadFromLocal("/tmp/rainbond.zip"); err != nil {
t.Fatal("downloadFromLocal error", err.Error())
}
t.Log(b.cacheDir)
}
func TestModify(t *testing.T) {
var b = BackupAPPRestore{
BackupID: "test",
EventID: "test",
serviceChange: make(map[string]Info, 0),
Logger: event.GetTestLogger(),
}
var appSnapshots = []*RegionServiceSnapshot{
&RegionServiceSnapshot{
ServiceID: "1234",
Service: &dbmodel.TenantServices{
ServiceID: "1234",
},
ServiceMntRelation: []*dbmodel.TenantServiceMountRelation{
&dbmodel.TenantServiceMountRelation{
ServiceID: "1234",
DependServiceID: "123456",
},
},
},
&RegionServiceSnapshot{
ServiceID: "123456",
Service: &dbmodel.TenantServices{
ServiceID: "1234",
},
ServiceEnv: []*dbmodel.TenantServiceEnvVar{
&dbmodel.TenantServiceEnvVar{
ServiceID: "123456",
Name: "testenv",
},
&dbmodel.TenantServiceEnvVar{
ServiceID: "123456",
Name: "testenv2",
},
},
},
}
b.modify(appSnapshots)
re, _ := ffjson.Marshal(appSnapshots)
t.Log(string(re))
}

3
builder/exector/plugin_image.go
View File

@ -22,7 +22,6 @@ import (
"fmt"
"strings"
"github.com/docker/engine-api/types"
"github.com/goodrain/rainbond/builder/sources"
"github.com/goodrain/rainbond/db"
@ -70,7 +69,7 @@ func (e *exectorManager) pluginImageBuild(in []byte) {
func (e *exectorManager) run(t *model.BuildPluginTaskBody, logger event.Logger) error {
if _, err := sources.ImagePull(e.DockerClient, t.ImageURL, types.ImagePullOptions{}, logger, 10); err != nil {
if _, err := sources.ImagePull(e.DockerClient, t.ImageURL, "", "", logger, 10); err != nil {
logrus.Errorf("pull image %v error, %v", t.ImageURL, err)
logger.Error("拉取镜像失败", map[string]string{"step": "builder-exector", "status": "failure"})
return err

16
builder/exector/plugin_image_test.go
View File

@ -17,19 +17,3 @@
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package exector
import (
"fmt"
"testing"
)
func TestSetTag(t *testing.T) {
curR := "goodrain.me"
image := "alpine"
alias := "gr123123"
mm, err := setTag(curR, image, alias)
if err != nil {
t.Error(err)
}
fmt.Println(mm)
}

3
builder/exector/share_image.go
View File

@ -25,7 +25,6 @@ import (
"github.com/Sirupsen/logrus"
"github.com/coreos/etcd/clientv3"
"github.com/docker/engine-api/client"
"github.com/docker/engine-api/types"
"github.com/goodrain/rainbond/builder/sources"
"github.com/goodrain/rainbond/event"
"github.com/pquerna/ffjson/ffjson"
@ -77,7 +76,7 @@ func NewImageShareItem(in []byte, DockerClient *client.Client, EtcdCli *clientv3
//ShareService ShareService
func (i *ImageShareItem) ShareService() error {
_, err := sources.ImagePull(i.DockerClient, i.LocalImageName, types.ImagePullOptions{}, i.Logger, 3)
_, err := sources.ImagePull(i.DockerClient, i.LocalImageName, "", "", i.Logger, 10)
if err != nil {
logrus.Errorf("pull image %s error: %s", i.LocalImageName, err.Error())
i.Logger.Error(fmt.Sprintf("拉取应用镜像: %s失败", i.LocalImageName), map[string]string{"step": "builder-exector", "status": "failure"})

3
builder/parser/docker_compose.go
View File

@ -30,7 +30,6 @@ import (
"github.com/goodrain/rainbond/event"
"github.com/docker/engine-api/client"
"github.com/docker/engine-api/types"
)
//DockerComposeParse docker compose 文件解析
@ -155,7 +154,7 @@ func (d *DockerComposeParse) Parse() ParseErrorList {
}
}
//获取镜像，验证是否存在
imageInspect, err := sources.ImagePull(d.dockerclient, service.image.String(), types.ImagePullOptions{}, d.logger, 5)
imageInspect, err := sources.ImagePull(d.dockerclient, service.image.String(), "", "", d.logger, 10)
if err != nil {
if strings.Contains(err.Error(), "No such image") {
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("镜像(%s)不存在", service.image.String()), SolveAdvice("modify_compose", "请确认ComposeFile输入镜像名是否正确")))

6
builder/parser/docker_run.go
View File

@ -31,7 +31,7 @@ import (
"github.com/docker/distribution/reference"
//"github.com/docker/docker/api/types"
"github.com/docker/engine-api/types"
//"github.com/docker/docker/client"
"github.com/docker/engine-api/client"
)
@ -94,7 +94,7 @@ func (d *DockerRunOrImageParse) Parse() ParseErrorList {
d.image = parseImageName(d.source)
}
//获取镜像，验证是否存在
imageInspect, err := sources.ImagePull(d.dockerclient, d.image.String(), types.ImagePullOptions{}, d.logger, 5)
imageInspect, err := sources.ImagePull(d.dockerclient, d.image.String(), "", "", d.logger, 10)
if err != nil {
if strings.Contains(err.Error(), "No such image") {
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("镜像(%s)不存在", d.image.String()), SolveAdvice("modify_image", "请确认输入镜像名是否正确")))
@ -192,7 +192,7 @@ func (d *DockerRunOrImageParse) dockerun(source []string) {
}
case "memory", "m":
d.memory = readmemory(s)
case "", "d", "i", "t", "it","P":
case "", "d", "i", "t", "it", "P":
d.image = parseImageName(s)
if len(source) > i+1 {
d.args = source[i+1:]

18
builder/sources/image.go
View File

@ -45,11 +45,25 @@ import (
//ImagePull 拉取镜像
//timeout 分钟为单位
func ImagePull(dockerCli *client.Client, image string, opts types.ImagePullOptions, logger event.Logger, timeout int) (*types.ImageInspect, error) {
func ImagePull(dockerCli *client.Client, image string, username, password string, logger event.Logger, timeout int) (*types.ImageInspect, error) {
if logger != nil {
//进度信息
logger.Info(fmt.Sprintf("开始获取镜像：%s", image), map[string]string{"step": "pullimage"})
}
var pullipo types.ImagePullOptions
if username != "" && password != "" {
auth, err := EncodeAuthToBase64(types.AuthConfig{Username: username, Password: password})
if err != nil {
logrus.Errorf("make auth base63 push image error: %s", err.Error())
logger.Error(fmt.Sprintf("生成获取镜像的Token失败"), map[string]string{"step": "builder-exector", "status": "failure"})
return nil, err
}
pullipo = types.ImagePullOptions{
RegistryAuth: auth,
}
} else {
pullipo = types.ImagePullOptions{}
}
rf, err := reference.ParseAnyReference(image)
if err != nil {
logrus.Errorf("reference image error: %s", err.Error())
@ -62,7 +76,7 @@ func ImagePull(dockerCli *client.Client, image string, opts types.ImagePullOptio
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*time.Duration(timeout))
defer cancel()
//TODO: 使用1.12版本api的bug “repository name must be canonical”，使用rf.String()完整的镜像地址
readcloser, err := dockerCli.ImagePull(ctx, rf.String(), opts)
readcloser, err := dockerCli.ImagePull(ctx, rf.String(), pullipo)
if err != nil {
logrus.Debugf("image name: %s readcloser error: %v", image, err.Error())
if strings.HasSuffix(err.Error(), "does not exist or no pull access") {

31
event/manager.go
View File

@ -19,6 +19,7 @@
package event
import (
"fmt"
"sync"
"sync/atomic"
"time"
@ -389,3 +390,33 @@ func (l *logger) send(message string, info map[string]string) {
util.SendNoBlocking(log, l.sendChan)
}
}
//GetTestLogger GetTestLogger
func GetTestLogger() Logger {
return &testLogger{}
}
type testLogger struct {
}
func (l *testLogger) GetChan() chan []byte {
return nil
}
func (l *testLogger) SetChan(ch chan []byte) {
}
func (l *testLogger) Event() string {
return "test"
}
func (l *testLogger) CreateTime() time.Time {
return time.Now()
}
func (l *testLogger) Info(message string, info map[string]string) {
fmt.Println("info:", message)
}
func (l *testLogger) Error(message string, info map[string]string) {
fmt.Println("error:", message)
}
func (l *testLogger) Debug(message string, info map[string]string) {
fmt.Println("debug:", message)
}

2
node/api/controller/K8SController.go
View File

@ -279,7 +279,7 @@ func GetNodeBasic(w http.ResponseWriter, r *http.Request) {
outRespSuccess(w, hostnode, nil)
}
//Resources 数据中心资源统计
//Resources specified node scheduler resources info
func Resources(w http.ResponseWriter, r *http.Request) {
result := new(model.Resource)
cpuR := 0

2
node/core/service/node_service.go
View File

@ -289,7 +289,7 @@ func (n *NodeService) GetNodeResource(nodeUID string) (*model.NodePodResource, *
return nil, err
}
if !node.Role.HasRule("compute") {
return nil, utils.CreateAPIHandleError(401, fmt.Errorf("node is not compute node"))
}
ps, error := k8s.GetPodsByNodeName(nodeUID)
if error != nil {

4
util/comman_test.go
View File

@ -50,13 +50,13 @@ func TestGetDirSizeByCmd(t *testing.T) {
}
func TestZip(t *testing.T) {
if err := Zip("/Users/qingguo/gopath/src/github.com/goodrain/rainbond", "/tmp/rainbond.zip"); err != nil {
if err := Zip("/tmp/test/", "/tmp/test.zip"); err != nil {
t.Fatal(err)
}
}
func TestUnzip(t *testing.T) {
if err := Unzip("/tmp/rainbond.zip", "/tmp/rainbond"); err != nil {
if err := Unzip("/tmp/test.aaa.zip", "/tmp/rainbond"); err != nil {
t.Fatal(err)
}
}

24
util/language.go
View File

@ -18,20 +18,28 @@
package util
import "os"
var translationMetadata = map[string]string{
"write console level metadata success": "写控制台级应用元数据成功",
"write region level metadata success": "写数据中心级应用元数据成功",
"Asynchronous tasks are sent successfully": "异步任务发送成功",
"create ftp client error": "创建FTP客户端失败",
"push slug file to local dir error": "上传应用代码包到本地目录失败",
"push slug file to sftp server error": "上传应用代码包到服务端失败",
"save image to local dir error": "保存镜像到本地目录失败",
"save image to hub error": "保存镜像到仓库失败",
"write console level metadata success": "写控制台级应用元数据成功",
"write region level metadata success": "写数据中心级应用元数据成功",
"Asynchronous tasks are sent successfully": "异步任务发送成功",
"create ftp client error": "创建FTP客户端失败",
"push slug file to local dir error": "上传应用代码包到本地目录失败",
"push slug file to sftp server error": "上传应用代码包到服务端失败",
"down slug file from sftp server error": "从服务端下载文件失败",
"save image to local dir error": "保存镜像到本地目录失败",
"save image to hub error": "保存镜像到仓库失败",
"Please try again or contact customer service": "后端服务开小差，请重试或联系客服",
"unzip metadata file error": "解压数据失败",
}
//Translation Translation English to Chinese
func Translation(english string) string {
if chinese, ok := translationMetadata[english]; ok {
if os.Getenv("RAINBOND_LANG") == "en" {
return english
}
return chinese
}
return english