goploy/service/SyncService.go
2021-03-22 14:10:15 +08:00

615 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package service
import (
"bytes"
"encoding/json"
"errors"
"github.com/zhenorzz/goploy/core"
"github.com/zhenorzz/goploy/model"
"github.com/zhenorzz/goploy/utils"
"github.com/zhenorzz/goploy/ws"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path"
"strconv"
"strings"
"sync"
)
// Gsync -
type Gsync struct {
UserInfo model.User
Project model.Project
ProjectServers model.ProjectServers
CommitID string
Branch string
}
type syncMessage struct {
serverName string
projectID int64
detail string
state int
}
// Exec Gsync
func (gsync Gsync) Exec() {
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(gsync.Project.ID, 10)+" deploy start")
publishTraceModel := model.PublishTrace{
Token: gsync.Project.LastPublishToken,
ProjectID: gsync.Project.ID,
ProjectName: gsync.Project.Name,
PublisherID: gsync.UserInfo.ID,
PublisherName: gsync.UserInfo.Name,
Type: model.Pull,
}
var gitCommitInfo utils.Commit
var err error
if len(gsync.CommitID) == 0 {
gitCommitInfo, err = gitFollow(gsync.Project, "origin/"+gsync.Project.Branch)
} else {
gitCommitInfo, err = gitFollow(gsync.Project, gsync.CommitID)
}
if err != nil {
gsync.Project.DeployFail()
publishTraceModel.Detail = err.Error()
publishTraceModel.State = model.Fail
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: gsync.Project.ID, ProjectName: gsync.Project.Name, State: ws.DeployFail, Message: err.Error()},
}
if _, err := publishTraceModel.AddRow(); err != nil {
core.Log(core.ERROR, err.Error())
}
notify(gsync.Project, gsync.ProjectServers, gitCommitInfo, model.ProjectFail, err.Error())
return
}
if gsync.Branch != "" {
gitCommitInfo.Branch = gsync.Branch
} else {
gitCommitInfo.Branch = "origin/" + gsync.Project.Branch
}
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{
ProjectID: gsync.Project.ID,
ProjectName: gsync.Project.Name,
State: ws.GitDone,
Message: "Get commit info",
Ext: gitCommitInfo,
},
}
ext, _ := json.Marshal(gitCommitInfo)
publishTraceModel.Ext = string(ext)
publishTraceModel.State = model.Success
if _, err := publishTraceModel.AddRow(); err != nil {
core.Log(core.ERROR, err.Error())
return
}
if totalFileNumber, err := (model.ProjectFile{ProjectID: gsync.Project.ID}).GetTotalByProjectID(); err != nil {
core.Log(core.ERROR, err.Error())
return
} else if totalFileNumber > 0 {
if err := utils.CopyDir(core.GetProjectFilePath(gsync.Project.ID), core.GetProjectPath(gsync.Project.ID)); err != nil {
core.Log(core.ERROR, err.Error())
return
}
}
if gsync.Project.AfterPullScript != "" {
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: gsync.Project.ID, ProjectName: gsync.Project.Name, State: ws.AfterPullScript, Message: "Run pull script"},
}
outputString, err := runAfterPullScript(gsync.Project)
publishTraceModel.Type = model.AfterPull
ext, _ := json.Marshal(struct {
Script string `json:"script"`
}{gsync.Project.AfterPullScript})
publishTraceModel.Ext = string(ext)
if err != nil {
gsync.Project.DeployFail()
publishTraceModel.Detail = err.Error()
publishTraceModel.State = model.Fail
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: gsync.Project.ID, ProjectName: gsync.Project.Name, State: ws.DeployFail, Message: err.Error()},
}
if _, err := publishTraceModel.AddRow(); err != nil {
core.Log(core.ERROR, err.Error())
}
notify(gsync.Project, gsync.ProjectServers, gitCommitInfo, model.ProjectFail, err.Error())
return
}
publishTraceModel.Detail = outputString
publishTraceModel.State = model.Success
if _, err := publishTraceModel.AddRow(); err != nil {
core.Log(core.ERROR, err.Error())
}
}
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: gsync.Project.ID, ProjectName: gsync.Project.Name, State: ws.Rsync, Message: "Rsync"},
}
ch := make(chan syncMessage, len(gsync.ProjectServers))
for _, projectServer := range gsync.ProjectServers {
go remoteSync(ch, gsync.UserInfo, gsync.Project, projectServer)
}
message := ""
for i := 0; i < len(gsync.ProjectServers); i++ {
syncMessage := <-ch
if syncMessage.state == model.ProjectFail {
message += syncMessage.serverName + " error message: " + syncMessage.detail
}
}
close(ch)
if message == "" {
gsync.Project.DeploySuccess()
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(gsync.Project.ID, 10)+" deploy success")
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: gsync.Project.ID, ProjectName: gsync.Project.Name, State: ws.DeploySuccess, Message: "Success"},
}
notify(gsync.Project, gsync.ProjectServers, gitCommitInfo, model.ProjectSuccess, message)
} else {
gsync.Project.DeployFail()
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(gsync.Project.ID, 10)+" deploy fail")
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: gsync.Project.ID, ProjectName: gsync.Project.Name, State: ws.DeployFail, Message: message},
}
notify(gsync.Project, gsync.ProjectServers, gitCommitInfo, model.ProjectFail, message)
}
if gsync.Project.SymlinkPath != "" {
var wg sync.WaitGroup
for _, projectServer := range gsync.ProjectServers {
wg.Add(1)
go func(projectServer model.ProjectServer) {
defer wg.Done()
removeExpiredBackup(gsync.Project, projectServer)
}(projectServer)
}
wg.Wait()
}
return
}
func gitFollow(project model.Project, target string) (utils.Commit, error) {
commit := utils.Commit{}
if err := (Repository{ProjectID: project.ID}.Create()); err != nil {
return commit, err
}
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: project.ID, ProjectName: project.Name, State: ws.GitCreate, Message: "git create"},
}
git := utils.GIT{Dir: core.GetProjectPath(project.ID)}
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: project.ID, ProjectName: project.Name, State: ws.GitClean, Message: "git clean"},
}
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(project.ID, 10)+" git add .")
if err := git.Add("."); err != nil {
core.Log(core.ERROR, err.Error()+", detail: "+git.Err.String())
return commit, errors.New(git.Err.String())
}
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(project.ID, 10)+" git reset --hard")
if err := git.Reset("--hard"); err != nil {
core.Log(core.ERROR, err.Error()+", detail: "+git.Err.String())
return commit, errors.New(git.Err.String())
}
// the length of commit id is 40
if len(target) != 40 {
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: project.ID, ProjectName: project.Name, State: ws.GitFetch, Message: "git fetch"},
}
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(project.ID, 10)+" git fetch")
if err := git.Fetch(); err != nil {
core.Log(core.ERROR, err.Error()+", detail: "+git.Err.String())
return commit, errors.New(git.Err.String())
}
}
ws.GetHub().Data <- &ws.Data{
Type: ws.TypeProject,
Message: ws.ProjectMessage{ProjectID: project.ID, ProjectName: project.Name, State: ws.GitCheckout, Message: "git checkout"},
}
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(project.ID, 10)+" git checkout -B goploy "+target)
if err := git.Checkout("-B", "goploy", target); err != nil {
core.Log(core.ERROR, err.Error()+", detail: "+git.Err.String())
return commit, errors.New(git.Err.String())
}
commit, err := gitCommitLog(project)
if err != nil {
return commit, err
}
return commit, err
}
func gitCommitLog(project model.Project) (utils.Commit, error) {
git := utils.GIT{Dir: core.GetProjectPath(project.ID)}
if err := git.Log("--stat", "--pretty=format:`start`%H`%an`%at`%s`%d`", "-n", "1"); err != nil {
core.Log(core.ERROR, err.Error()+", detail: "+git.Err.String())
return utils.Commit{}, errors.New(git.Err.String())
}
commitList := utils.ParseGITLog(git.Output.String())
return commitList[0], nil
}
func runAfterPullScript(project model.Project) (string, error) {
srcPath := core.GetProjectPath(project.ID)
scriptName := "goploy-after-pull." + utils.GetScriptExt(project.AfterPullScriptMode)
scriptFullName := path.Join(srcPath, scriptName)
scriptMode := "bash"
if len(project.AfterPullScriptMode) != 0 {
scriptMode = project.AfterPullScriptMode
}
ioutil.WriteFile(scriptFullName, []byte(project.AfterPullScript), 0755)
handler := exec.Command(scriptMode, scriptFullName)
handler.Dir = srcPath
var outbuf, errbuf bytes.Buffer
handler.Stdout = &outbuf
handler.Stderr = &errbuf
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(project.ID, 10)+project.AfterPullScript)
if err := handler.Run(); err != nil {
core.Log(core.ERROR, errbuf.String())
return "", errors.New(errbuf.String())
}
os.Remove(scriptName)
return outbuf.String(), nil
}
func remoteSync(msgChIn chan<- syncMessage, userInfo model.User, project model.Project, projectServer model.ProjectServer) {
remoteMachine := projectServer.ServerOwner + "@" + projectServer.ServerIP
destDir := project.Path
ext, _ := json.Marshal(struct {
ServerID int64 `json:"serverId"`
ServerName string `json:"serverName"`
}{projectServer.ServerID, projectServer.ServerName})
publishTraceModel := model.PublishTrace{
Token: project.LastPublishToken,
ProjectID: project.ID,
ProjectName: project.Name,
PublisherID: userInfo.ID,
PublisherName: userInfo.Name,
Type: model.Deploy,
Ext: string(ext),
}
// write after deploy script for rsync
if len(project.AfterDeployScript) != 0 {
scriptName := path.Join(core.GetProjectPath(project.ID), "goploy-after-deploy."+utils.GetScriptExt(project.AfterDeployScriptMode))
ioutil.WriteFile(scriptName, []byte(replaceScriptVars(project.AfterDeployScript, project)), 0755)
}
rsyncOption, _ := utils.ParseCommandLine(project.RsyncOption)
rsyncOption = append([]string{"--exclude", "goploy-after-pull.sh", "--include", "goploy-after-deploy.sh"}, rsyncOption...)
rsyncOption = append(rsyncOption, "-e", "ssh -p "+strconv.Itoa(projectServer.ServerPort)+" -o StrictHostKeyChecking=no")
if len(project.SymlinkPath) != 0 {
destDir = path.Join(project.SymlinkPath, project.Name, project.LastPublishToken)
rsyncOption = append(rsyncOption, "--rsync-path=mkdir -p "+destDir+" && rsync")
}
srcPath := core.GetProjectPath(project.ID) + "/"
// rsync path can not contain colon
// windows like C:\
if strings.Contains(srcPath, ":\\") {
srcPath = "/cygdrive/" + strings.Replace(srcPath, ":\\", "/", 1)
}
destPath := remoteMachine + ":" + destDir
rsyncOption = append(rsyncOption, srcPath, destPath)
cmd := exec.Command("rsync", rsyncOption...)
var outbuf, errbuf bytes.Buffer
cmd.Stdout = &outbuf
cmd.Stderr = &errbuf
core.Log(core.TRACE, "projectID:"+strconv.FormatInt(project.ID, 10)+" rsync "+strings.Join(rsyncOption, " "))
if err := cmd.Run(); err != nil {
core.Log(core.ERROR, "err: "+err.Error()+", detail: "+errbuf.String())
publishTraceModel.Detail = "err: " + err.Error() + ", detail: " + errbuf.String()
publishTraceModel.State = model.Fail
publishTraceModel.AddRow()
msgChIn <- syncMessage{
serverName: projectServer.ServerName,
projectID: project.ID,
detail: "err: " + err.Error() + ", detail: " + errbuf.String(),
state: model.ProjectFail,
}
return
}
ext, _ = json.Marshal(struct {
ServerID int64 `json:"serverId"`
ServerName string `json:"serverName"`
Command string `json:"command"`
}{projectServer.ServerID, projectServer.ServerName, "rsync " + strings.Join(rsyncOption, " ")})
publishTraceModel.Ext = string(ext)
publishTraceModel.Detail = outbuf.String()
publishTraceModel.State = model.Success
publishTraceModel.AddRow()
var afterDeployCommands []string
if len(project.SymlinkPath) != 0 {
afterDeployCommands = append(afterDeployCommands, "ln -sfn "+destDir+" "+project.Path)
// change the destination folder time, make sure it can not be clean
afterDeployCommands = append(afterDeployCommands, "touch -m "+destDir)
}
if len(project.AfterDeployScript) != 0 {
scriptMode := "bash"
if len(project.AfterDeployScriptMode) != 0 {
scriptMode = project.AfterDeployScriptMode
}
afterDeployScriptPath := path.Join(project.Path, "goploy-after-deploy."+utils.GetScriptExt(project.AfterDeployScriptMode))
afterDeployCommands = append(afterDeployCommands, scriptMode+" "+afterDeployScriptPath)
afterDeployCommands = append(afterDeployCommands, "rm -f "+afterDeployScriptPath)
}
// no symlink and deploy script
if len(afterDeployCommands) == 0 {
msgChIn <- syncMessage{
serverName: projectServer.ServerName,
projectID: project.ID,
state: model.ProjectSuccess,
}
return
}
publishTraceModel.Type = model.AfterDeploy
ext, _ = json.Marshal(struct {
ServerID int64 `json:"serverId"`
ServerName string `json:"serverName"`
Script string `json:"script"`
}{projectServer.ServerID, projectServer.ServerName, strings.Join(afterDeployCommands, ";")})
publishTraceModel.Ext = string(ext)
client, dialError := utils.DialSSH(projectServer.ServerOwner, projectServer.ServerPassword, projectServer.ServerPath, projectServer.ServerIP, projectServer.ServerPort)
if dialError != nil {
core.Log(core.ERROR, dialError.Error())
publishTraceModel.Detail = dialError.Error()
publishTraceModel.State = model.Fail
publishTraceModel.AddRow()
msgChIn <- syncMessage{
serverName: projectServer.ServerName,
projectID: project.ID,
detail: dialError.Error(),
state: model.ProjectFail,
}
return
}
session, sessionErr := client.NewSession()
if sessionErr != nil {
core.Log(core.ERROR, sessionErr.Error())
publishTraceModel.Detail = sessionErr.Error()
publishTraceModel.State = model.Fail
publishTraceModel.AddRow()
msgChIn <- syncMessage{
serverName: projectServer.ServerName,
projectID: project.ID,
detail: sessionErr.Error(),
state: model.ProjectFail,
}
return
}
var sshOutbuf, sshErrbuf bytes.Buffer
session.Stdout = &sshOutbuf
session.Stderr = &sshErrbuf
if err := session.Run(strings.Join(afterDeployCommands, ";")); err != nil {
core.Log(core.ERROR, "err: "+err.Error()+", detail: "+sshErrbuf.String())
publishTraceModel.Detail = err.Error()
publishTraceModel.State = model.Fail
publishTraceModel.AddRow()
msgChIn <- syncMessage{
serverName: projectServer.ServerName,
projectID: project.ID,
detail: err.Error(),
state: model.ProjectFail,
}
return
}
publishTraceModel.Detail = sshOutbuf.String()
publishTraceModel.State = model.Success
publishTraceModel.AddRow()
defer session.Close()
msgChIn <- syncMessage{
serverName: projectServer.ServerName,
projectID: project.ID,
state: model.ProjectSuccess,
}
return
}
// commit id
// commit message
// server ip & name
// deploy user name
// deploy time
func notify(project model.Project, projectServers model.ProjectServers, commitInfo utils.Commit, deployState int, detail string) {
serverList := ""
for _, projectServer := range projectServers {
if projectServer.ServerName != projectServer.ServerIP {
serverList += projectServer.ServerName + "(" + projectServer.ServerIP + ")"
} else {
serverList += projectServer.ServerIP
}
serverList += ", "
}
serverList = strings.TrimRight(serverList, ", ")
if project.NotifyType == model.NotifyWeiXin {
type markdown struct {
Content string `json:"content"`
}
type message struct {
Msgtype string `json:"msgtype"`
Markdown markdown `json:"markdown"`
}
content := "Deploy: <font color=\"warning\">" + project.Name + "</font>\n"
content += "Publisher: <font color=\"comment\">" + project.PublisherName + "</font>\n"
content += "Branch: <font color=\"comment\">" + commitInfo.Branch + "</font>\n"
content += "CommitSHA: <font color=\"comment\">" + commitInfo.Commit + "</font>\n"
content += "CommitMessage: <font color=\"comment\">" + commitInfo.Message + "</font>\n"
content += "ServerList: <font color=\"comment\">" + serverList + "</font>\n"
if deployState == model.ProjectFail {
content += "State: <font color=\"red\">fail</font> \n"
content += "> Detail: <font color=\"comment\">" + detail + "</font>"
} else {
content += "State: <font color=\"green\">success</font>"
}
msg := message{
Msgtype: "markdown",
Markdown: markdown{
Content: content,
},
}
b, _ := json.Marshal(msg)
_, err := http.Post(project.NotifyTarget, "application/json", bytes.NewBuffer(b))
if err != nil {
core.Log(core.ERROR, "projectID:"+strconv.FormatInt(project.ID, 10)+" "+err.Error())
}
} else if project.NotifyType == model.NotifyDingTalk {
type markdown struct {
Title string `json:"title"`
Text string `json:"text"`
}
type message struct {
Msgtype string `json:"msgtype"`
Markdown markdown `json:"markdown"`
}
text := "#### Deploy" + project.Name + " \n "
text += "#### Publisher" + project.PublisherName + " \n "
text += "#### Branch" + commitInfo.Branch + " \n "
text += "#### CommitSHA" + commitInfo.Commit + " \n "
text += "#### CommitMessage" + commitInfo.Message + " \n "
text += "#### ServerList" + serverList + " \n "
if deployState == model.ProjectFail {
text += "#### State <font color=\"red\">fail</font> \n "
text += "> Detail: <font color=\"comment\">" + detail + "</font>"
} else {
text += "#### State <font color=\"green\">success</font>"
}
msg := message{
Msgtype: "markdown",
Markdown: markdown{
Title: project.Name,
Text: text,
},
}
b, _ := json.Marshal(msg)
_, err := http.Post(project.NotifyTarget, "application/json", bytes.NewBuffer(b))
if err != nil {
core.Log(core.ERROR, "projectID:"+strconv.FormatInt(project.ID, 10)+" "+err.Error())
}
} else if project.NotifyType == model.NotifyFeiShu {
type message struct {
Title string `json:"title"`
Text string `json:"text"`
}
text := ""
text += "Publisher: " + project.PublisherName + "\n"
text += "Branch: " + commitInfo.Branch + "\n"
text += "CommitSHA: " + commitInfo.Commit + "\n"
text += "CommitMessage: " + commitInfo.Message + "\n"
text += "ServerList: " + serverList + "\n"
if deployState == model.ProjectFail {
text += "State: fail\n "
text += "Detail: " + detail
} else {
text += "State: success"
}
msg := message{
Title: "Deploy:" + project.Name,
Text: text,
}
b, _ := json.Marshal(msg)
_, err := http.Post(project.NotifyTarget, "application/json", bytes.NewBuffer(b))
if err != nil {
core.Log(core.ERROR, "projectID:"+strconv.FormatInt(project.ID, 10)+" "+err.Error())
}
} else if project.NotifyType == model.NotifyCustom {
type message struct {
Code int `json:"code"`
Message string `json:"message"`
Data struct {
ProjectID int64 `json:"projectId"`
ProjectName string `json:"projectName"`
Publisher string `json:"publisher"`
Branch string `json:"branch"`
CommitSHA string `json:"commitSHA"`
CommitMessage string `json:"commitMessage"`
ServerList string `json:"serverList"`
} `json:"data"`
}
code := 0
if deployState == model.ProjectFail {
code = 1
}
msg := message{
Code: code,
Message: detail,
}
msg.Data.ProjectID = project.ID
msg.Data.ProjectName = project.Name
msg.Data.Publisher = project.PublisherName
msg.Data.Branch = commitInfo.Branch
msg.Data.CommitSHA = commitInfo.Commit
msg.Data.CommitMessage = commitInfo.Message
msg.Data.ServerList = serverList
b, _ := json.Marshal(msg)
_, err := http.Post(project.NotifyTarget, "application/json", bytes.NewBuffer(b))
if err != nil {
core.Log(core.ERROR, "projectID:"+strconv.FormatInt(project.ID, 10)+" "+err.Error())
}
}
}
//keep the latest 10 project
func removeExpiredBackup(project model.Project, projectServer model.ProjectServer) {
client, err := utils.DialSSH(projectServer.ServerOwner, projectServer.ServerPassword, projectServer.ServerPath, projectServer.ServerIP, projectServer.ServerPort)
if err != nil {
core.Log(core.ERROR, err.Error())
return
}
session, err := client.NewSession()
if err != nil {
core.Log(core.ERROR, err.Error())
return
}
defer session.Close()
var sshOutbuf, sshErrbuf bytes.Buffer
session.Stdout = &sshOutbuf
session.Stderr = &sshErrbuf
destDir := path.Join(project.SymlinkPath, project.Name)
if err = session.Run("cd " + destDir + ";ls -t | awk 'NR>10' | xargs rm -rf"); err != nil {
core.Log(core.ERROR, err.Error())
}
}
func replaceScriptVars(script string, project model.Project) string {
scriptVars := map[string]string{
"${PROJECT_PATH}": project.Path,
"${PROJECT_SYMLINK_PATH}": path.Join(project.SymlinkPath, project.LastPublishToken),
"${PROJECT_NAME}": project.Name,
}
for key, value := range scriptVars {
script = strings.Replace(script, key, value, -1)
}
return script
}