support oss code source

This commit is contained in:
barnettZQG 2021-07-28 21:31:58 +08:00
parent 501b588445
commit efc2acaa07
12 changed files with 242 additions and 146 deletions

View File

@ -25,6 +25,7 @@ import (
"github.com/goodrain/rainbond/builder"
"github.com/goodrain/rainbond/builder/parser/code"
"github.com/goodrain/rainbond/builder/sources"
"github.com/goodrain/rainbond/event"
"k8s.io/client-go/kubernetes"
@ -43,6 +44,7 @@ func init() {
buildcreaters[code.Python] = slugBuilder
buildcreaters[code.Nodejs] = slugBuilder
buildcreaters[code.Golang] = slugBuilder
buildcreaters[code.OSS] = slugBuilder
}
var buildcreaters map[code.Lang]CreaterBuild
@ -82,6 +84,7 @@ type Request struct {
CacheDir string
TGZDir string
RepositoryURL string
CodeSouceInfo sources.CodeSourceInfo
Branch string
ServiceAlias string
ServiceID string

View File

@ -19,11 +19,9 @@
package build
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
@ -33,14 +31,11 @@ import (
"github.com/docker/docker/api/types"
"github.com/eapache/channels"
"github.com/fsnotify/fsnotify"
"github.com/goodrain/rainbond/builder"
jobc "github.com/goodrain/rainbond/builder/job"
"github.com/goodrain/rainbond/builder/parser/code"
"github.com/goodrain/rainbond/builder/sources"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/util"
"github.com/pquerna/ffjson/ffjson"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -53,7 +48,6 @@ func slugBuilder() (Build, error) {
type slugBuild struct {
tgzDir string
buildCacheDir string
sourceDir string
re *Request
}
@ -107,8 +101,14 @@ func (s *slugBuild) Build(re *Request) (*Response, error) {
func (s *slugBuild) writeRunDockerfile(sourceDir, packageName string, envs map[string]string) error {
runDockerfile := `
FROM %s
ARG CODE_COMMIT_HASH
ARG CODE_COMMIT_USER
ARG CODE_COMMIT_MESSAGE
COPY %s /tmp/slug/slug.tgz
RUN chown rain:rain /tmp/slug/slug.tgz
ENV CODE_COMMIT_HASH=${CODE_COMMIT_HASH}
ENV CODE_COMMIT_USER=${CODE_COMMIT_USER}
ENV CODE_COMMIT_MESSAGE=${CODE_COMMIT_MESSAGE}
ENV VERSION=%s
`
result := util.ParseVariable(fmt.Sprintf(runDockerfile, builder.RUNNERIMAGENAME, packageName, s.re.DeployVersion), envs)
@ -138,6 +138,11 @@ func (s *slugBuild) buildRunnerImage(slugPackage string) (string, error) {
}
//build runtime image
runbuildOptions := types.ImageBuildOptions{
BuildArgs: map[string]*string{
"CODE_COMMIT_HASH": &s.re.Commit.Hash,
"CODE_COMMIT_USER": &s.re.Commit.User,
"CODE_COMMIT_MESSAGE": &s.re.Commit.Message,
},
Tags: []string{imageName},
Remove: true,
}
@ -178,56 +183,6 @@ func (s *slugBuild) buildRunnerImage(slugPackage string) (string, error) {
return imageName, nil
}
func (s *slugBuild) readLogFile(logfile string, logger event.Logger, closed chan struct{}) {
file, _ := os.Open(logfile)
watcher, _ := fsnotify.NewWatcher()
defer watcher.Close()
_ = watcher.Add(logfile)
readerr := bufio.NewReader(file)
for {
line, _, err := readerr.ReadLine()
if err != nil {
if err != io.EOF {
logrus.Errorf("Read build container log error:%s", err.Error())
return
}
wait := func() error {
for {
select {
case <-closed:
return nil
case evt := <-watcher.Events:
if evt.Op&fsnotify.Write == fsnotify.Write {
return nil
}
case err := <-watcher.Errors:
return err
}
}
}
if err := wait(); err != nil {
logrus.Errorf("Read build container log error:%s", err.Error())
return
}
}
if logger != nil {
var message = make(map[string]string)
if err := ffjson.Unmarshal(line, &message); err == nil {
if m, ok := message["log"]; ok {
logger.Info(m, map[string]string{"step": "build-exector"})
}
} else {
fmt.Println(err.Error())
}
}
select {
case <-closed:
return
default:
}
}
}
func (s *slugBuild) getSourceCodeTarFile(re *Request) (string, error) {
var cmd []string
sourceTarFile := fmt.Sprintf("%s/%s-%s.tar", util.GetParentDirectory(re.SourceDir), re.ServiceID, re.DeployVersion)
@ -253,7 +208,7 @@ func (s *slugBuild) stopPreBuildJob(re *Request) error {
if err != nil {
logrus.Errorf("get pre build job for service %s failure ,%s", re.ServiceID, err.Error())
}
if jobList != nil && len(jobList) > 0 {
if len(jobList) > 0 {
for _, job := range jobList {
jobc.GetJobController().DeleteJob(job.Name)
}
@ -263,6 +218,7 @@ func (s *slugBuild) stopPreBuildJob(re *Request) error {
func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string) (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) {
slugSubPath := strings.TrimPrefix(re.TGZDir, "/grdata/")
lazyloading := sourceTarFileName == ""
sourceTarPath := strings.TrimPrefix(sourceTarFileName, "/cache/")
cacheSubPath := strings.TrimPrefix(re.CacheDir, "/cache/")
@ -279,10 +235,12 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
MountPath: "/tmp/slug",
SubPath: slugSubPath,
},
{
}
if !lazyloading {
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "source-file",
MountPath: "/tmp/app-source.tar",
},
})
}
volumes = []corev1.Volume{
{
@ -302,7 +260,9 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
},
},
},
{
}
if !lazyloading {
volumes = append(volumes, corev1.Volume{
Name: "source-file",
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
@ -311,7 +271,7 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
Type: &unset,
},
},
},
})
}
} else {
volumes = []corev1.Volume{
@ -343,11 +303,13 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
MountPath: "/tmp/slug",
SubPath: slugSubPath,
},
{
}
if !lazyloading {
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "app",
MountPath: "/tmp/app-source.tar",
SubPath: sourceTarPath,
},
})
}
}
return volumes, volumeMounts
@ -357,16 +319,21 @@ func (s *slugBuild) runBuildJob(re *Request) error {
//prepare build code dir
re.Logger.Info(util.Translation("Start make code package"), map[string]string{"step": "build-exector"})
start := time.Now()
sourceTarFileName, err := s.getSourceCodeTarFile(re)
if err != nil {
return fmt.Errorf("create source code tar file error:%s", err.Error())
var sourceTarFileName string
if re.ServerType != "oss" {
var err error
sourceTarFileName, err = s.getSourceCodeTarFile(re)
if err != nil {
return fmt.Errorf("create source code tar file error:%s", err.Error())
}
// remove source cache tar file
defer func() {
os.Remove(sourceTarFileName)
}()
}
re.Logger.Info(util.Translation("make code package success"), map[string]string{"step": "build-exector"})
logrus.Infof("package code for building service %s version %s successful, take time %s", re.ServiceID, re.DeployVersion, time.Now().Sub(start))
// remove source cache tar file
defer func() {
os.Remove(sourceTarFileName)
}()
name := fmt.Sprintf("%s-%s", re.ServiceID, re.DeployVersion)
namespace := re.RbdNamespace
job := corev1.Pod{
@ -383,8 +350,16 @@ func (s *slugBuild) runBuildJob(re *Request) error {
{Name: "SLUG_VERSION", Value: re.DeployVersion},
{Name: "SERVICE_ID", Value: re.ServiceID},
{Name: "TENANT_ID", Value: re.TenantID},
{Name: "CODE_COMMIT_HASH", Value: re.Commit.Hash},
{Name: "CODE_COMMIT_USER", Value: re.Commit.User},
{Name: "CODE_COMMIT_MESSAGE", Value: re.Commit.Message},
{Name: "LANGUAGE", Value: re.Lang.String()},
}
if re.ServerType == "oss" {
envs = append(envs, corev1.EnvVar{Name: "PACKAGE_DOWNLOAD_URL", Value: re.RepositoryURL})
envs = append(envs, corev1.EnvVar{Name: "PACKAGE_DOWNLOAD_USER", Value: re.CodeSouceInfo.User})
envs = append(envs, corev1.EnvVar{Name: "PACKAGE_DOWNLOAD_PASS", Value: re.CodeSouceInfo.Password})
}
var mavenSettingName string
for k, v := range re.BuildEnvs {
if k == "MAVEN_SETTING_NAME" {
@ -489,7 +464,7 @@ func (s *slugBuild) runBuildJob(re *Request) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
logrus.Debugf("create job[name: %s; namespace: %s]", job.Name, job.Namespace)
err = jobc.GetJobController().ExecJob(ctx, &job, writer, reChan)
err := jobc.GetJobController().ExecJob(ctx, &job, writer, reChan)
if err != nil {
logrus.Errorf("create new job:%s failed: %s", name, err.Error())
return err

View File

@ -11,12 +11,13 @@ import (
"testing"
"time"
"github.com/goodrain/rainbond/builder/parser/code"
"github.com/goodrain/rainbond/cmd/builder/option"
"github.com/goodrain/rainbond/event"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
jobc "github.com/goodrain/rainbond/builder/job"
"github.com/goodrain/rainbond/builder/parser/code"
"github.com/goodrain/rainbond/builder/sources"
"github.com/goodrain/rainbond/cmd/builder/option"
"github.com/goodrain/rainbond/event"
etcdutil "github.com/goodrain/rainbond/util/etcd"
k8sutil "github.com/goodrain/rainbond/util/k8s"
@ -126,3 +127,46 @@ func TestDockerClient(t *testing.T) {
// t.Log("image is : ", image.ID)
// }
}
func TestBuildFromOSS(t *testing.T) {
restConfig, err := k8sutil.NewRestConfig("/Users/barnett/.kube/config")
if err != nil {
t.Fatal(err)
}
os.Setenv("IMAGE_PULL_SECRET", "rbd-hub-credentials")
clientset, err := kubernetes.NewForConfig(restConfig)
if err != nil {
t.Fatal(err)
}
stop := make(chan struct{})
if err := jobc.InitJobController("rbd-system", stop, clientset); err != nil {
t.Fatal(err)
}
logger := event.GetTestLogger()
req := &Request{
ServerType: "oss",
RepositoryURL: "http://8081.gr021644.64q1jlfb.17f4cc.grapps.cn/artifactory/dev/java-war-demo-master.zip",
CodeSouceInfo: sources.CodeSourceInfo{
User: "demo",
Password: "gr123465!",
},
KubeClient: clientset,
Ctx: context.Background(),
ServiceID: "d9b8d718510dc53118af1e1219e36d3a",
DeployVersion: "123asdadsadsasdasd1",
TenantID: "7c89455140284fd7b263038b44dc65bc",
Lang: code.OSS,
Logger: logger,
GRDataPVCName: "rbd-cpt-grdata",
CachePVCName: "rbd-chaos-cache",
}
build, err := GetBuild(code.OSS)
if err != nil {
t.Fatal(err)
}
res, err := build.Build(req)
if err != nil {
t.Fatal(err)
}
t.Log(res.MediumPath)
}

View File

@ -40,9 +40,8 @@ import (
"github.com/goodrain/rainbond/util"
"github.com/pquerna/ffjson/ffjson"
"github.com/sirupsen/logrus"
"github.com/tidwall/gjson" //"github.com/docker/docker/api/types"
"github.com/tidwall/gjson"
"k8s.io/client-go/kubernetes"
//"github.com/docker/docker/client"
)
//SourceCodeBuildItem SouceCodeBuildItem
@ -59,8 +58,7 @@ type SourceCodeBuildItem struct {
Logger event.Logger `json:"logger"`
EventID string `json:"event_id"`
CacheDir string `json:"cache_dir"`
//SourceDir string `json:"source_dir"`
TGZDir string `json:"tgz_dir"`
TGZDir string `json:"tgz_dir"`
DockerClient *client.Client
KubeClient kubernetes.Interface
RbdNamespace string
@ -139,7 +137,7 @@ func (i *SourceCodeBuildItem) Run(timeout time.Duration) error {
i.RepoInfo = rbi
if err := i.prepare(); err != nil {
logrus.Errorf("prepare build code error: %s", err.Error())
i.Logger.Error(fmt.Sprintf("准备源码构建失败"), map[string]string{"step": "builder-exector", "status": "failure"})
i.Logger.Error("准备源码构建失败", map[string]string{"step": "builder-exector", "status": "failure"})
return err
}
i.CodeSouceInfo.RepositoryURL = rbi.RepostoryURL
@ -163,19 +161,21 @@ func (i *SourceCodeBuildItem) Run(timeout time.Duration) error {
Message: rs.Logs.CommitEntrys[0].Msg,
Author: rs.Logs.CommitEntrys[0].Author,
}
case "oss":
i.commit = Commit{}
default:
//default git
rs, err := sources.GitCloneOrPull(i.CodeSouceInfo, rbi.GetCodeHome(), i.Logger, 5)
if err != nil {
logrus.Errorf("pull git code error: %s", err.Error())
i.Logger.Error(fmt.Sprintf("拉取代码失败,请确保代码可以被正常下载"), map[string]string{"step": "builder-exector", "status": "failure"})
i.Logger.Error("拉取代码失败,请确保代码可以被正常下载", map[string]string{"step": "builder-exector", "status": "failure"})
return err
}
//get last commit
commit, err := sources.GetLastCommit(rs)
if err != nil || commit == nil {
logrus.Errorf("get code commit info error: %s", err.Error())
i.Logger.Error(fmt.Sprintf("读取代码版本信息失败"), map[string]string{"step": "builder-exector", "status": "failure"})
i.Logger.Error("读取代码版本信息失败", map[string]string{"step": "builder-exector", "status": "failure"})
return err
}
i.commit = Commit{
@ -241,6 +241,7 @@ func (i *SourceCodeBuildItem) codeBuild() (*build.Response, error) {
CacheDir: i.CacheDir,
TGZDir: i.TGZDir,
RepositoryURL: i.RepoInfo.RepostoryURL,
CodeSouceInfo: i.CodeSouceInfo,
ServiceAlias: i.ServiceAlias,
ServiceID: i.ServiceID,
TenantID: i.TenantID,
@ -265,21 +266,6 @@ func (i *SourceCodeBuildItem) codeBuild() (*build.Response, error) {
return res, err
}
func (i *SourceCodeBuildItem) getExtraHosts() (extraHosts []string, err error) {
endpoints, err := i.KubeClient.CoreV1().Endpoints(i.RbdNamespace).Get(context.Background(), i.RbdRepoName, metav1.GetOptions{})
if err != nil {
logrus.Errorf("do not found ep by name: %s in namespace: %s", i.RbdRepoName, i.Namespace)
return nil, err
}
for _, subset := range endpoints.Subsets {
for _, addr := range subset.Addresses {
extraHosts = append(extraHosts, fmt.Sprintf("maven.goodrain.me:%s", addr.IP))
extraHosts = append(extraHosts, fmt.Sprintf("lang.goodrain.me:%s", addr.IP))
}
}
return
}
func (i *SourceCodeBuildItem) getHostAlias() (hostAliasList []build.HostAlias, err error) {
endpoints, err := i.KubeClient.CoreV1().Endpoints(i.RbdNamespace).Get(context.Background(), i.RbdRepoName, metav1.GetOptions{})
if err != nil {

View File

@ -287,7 +287,7 @@ func (e *exectorManager) buildFromImage(task *pb.TaskMessage) {
}()
start := time.Now()
defer func() {
logrus.Debugf("complete build from source code, consuming time %s", time.Now().Sub(start).String())
logrus.Debugf("complete build from source code, consuming time %s", time.Since(start).String())
}()
for n := 0; n < 2; n++ {
err := i.Run(time.Minute * 30)

View File

@ -125,14 +125,12 @@ func (e *exectorManager) serviceCheck(task *pb.TaskMessage) {
return
}
errList := pr.Parse()
if errList != nil {
for i, err := range errList {
if err.SolveAdvice == "" && input.SourceType != "sourcecode" {
errList[i].SolveAdvice = fmt.Sprintf("解析器认为镜像名为:%s,请确认是否正确或镜像是否存在", pr.GetImage())
}
if err.SolveAdvice == "" && input.SourceType == "sourcecode" {
errList[i].SolveAdvice = fmt.Sprintf("源码智能解析失败,请联系客服")
}
for i, err := range errList {
if err.SolveAdvice == "" && input.SourceType != "sourcecode" {
errList[i].SolveAdvice = fmt.Sprintf("解析器认为镜像名为:%s,请确认是否正确或镜像是否存在", pr.GetImage())
}
if err.SolveAdvice == "" && input.SourceType == "sourcecode" {
errList[i].SolveAdvice = "源码智能解析失败"
}
}
serviceInfos := pr.GetServiceInfo()

View File

@ -115,6 +115,9 @@ var Grails Lang = "Grails"
//NetCore Lang
var NetCore Lang = ".NetCore"
//OSS Lang
var OSS Lang = "OSS"
//GetLangType check code lang
func GetLangType(homepath string) (Lang, error) {
if ok, _ := util.FileExists(homepath); !ok {

View File

@ -19,13 +19,15 @@
package parser
import (
"context"
"encoding/base64"
"fmt"
"os"
"path"
"runtime"
"strconv"
"strings"
"github.com/docker/docker/client"
"github.com/goodrain/rainbond/builder"
"github.com/goodrain/rainbond/builder/parser/code"
multi "github.com/goodrain/rainbond/builder/parser/code/multisvc"
@ -34,6 +36,7 @@ import (
"github.com/goodrain/rainbond/db/model"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/util"
"github.com/melbahja/got"
"github.com/pquerna/ffjson/ffjson"
"github.com/sirupsen/logrus"
"gopkg.in/src-d/go-git.v4/plumbing"
@ -42,18 +45,17 @@ import (
//SourceCodeParse docker run 命令解析或直接镜像名解析
type SourceCodeParse struct {
ports map[int]*types.Port
volumes map[string]*types.Volume
envs map[string]*types.Env
source string
memory int
image Image
args []string
branchs []string
errors []ParseError
dockerclient *client.Client
logger event.Logger
Lang code.Lang
ports map[int]*types.Port
volumes map[string]*types.Volume
envs map[string]*types.Env
source string
memory int
image Image
args []string
branchs []string
errors []ParseError
logger event.Logger
Lang code.Lang
Runtime bool `json:"runtime"`
Dependencies bool `json:"dependencies"`
@ -83,6 +85,7 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
d.errappend(Errorf(FatalError, "source can not be empty"))
return d.errors
}
logrus.Debugf("component source check info: %s", d.source)
var csi sources.CodeSourceInfo
err := ffjson.Unmarshal([]byte(d.source), &csi)
if err != nil {
@ -105,6 +108,14 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
d.errappend(ErrorAndSolve(FatalError, "Git项目仓库地址格式错误", SolveAdvice("modify_url", "请确认并修改仓库地址")))
return d.errors
}
// The source code is useless after the test is completed, and needs to be deleted.
defer func() {
if sources.CheckFileExist(buildInfo.GetCodeHome()) {
if err := sources.RemoveDir(buildInfo.GetCodeHome()); err != nil {
logrus.Warningf("remove source code: %v", err)
}
}
}()
gitFunc := func() ParseErrorList {
//get code
if !util.DirIsEmpty(buildInfo.GetCodeHome()) {
@ -131,26 +142,26 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
}
if err == transport.ErrRepositoryNotFound {
solve := SolveAdvice("modify_repo", "请确认仓库地址是否正确")
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("Git项目仓库不存在"), solve))
d.errappend(ErrorAndSolve(FatalError, "Git项目仓库不存在", solve))
return d.errors
}
if err == transport.ErrEmptyRemoteRepository {
solve := SolveAdvice("open_repo", "请确认已提交代码")
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("Git项目仓库无有效文件"), solve))
d.errappend(ErrorAndSolve(FatalError, "Git项目仓库无有效文件", solve))
return d.errors
}
if strings.Contains(err.Error(), "ssh: unable to authenticate") {
solve := SolveAdvice("get_publickey", "请获取授权Key配置到你的仓库项目试试")
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("远程仓库SSH验证错误"), solve))
d.errappend(ErrorAndSolve(FatalError, "远程仓库SSH验证错误", solve))
return d.errors
}
if strings.Contains(err.Error(), "context deadline exceeded") {
solve := "请确认源码仓库能否正常访问"
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("获取代码超时"), solve))
d.errappend(ErrorAndSolve(FatalError, "获取代码超时", solve))
return d.errors
}
logrus.Errorf("git clone error,%s", err.Error())
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("获取代码失败"), "请确认仓库能否正常访问,或联系客服咨询"))
d.errappend(ErrorAndSolve(FatalError, "获取代码失败"+err.Error(), "请确认仓库能否正常访问。"))
return d.errors
}
//获取分支
@ -172,7 +183,6 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
svnFunc := func() ParseErrorList {
if sources.CheckFileExist(buildInfo.GetCodeHome()) {
if err := sources.RemoveDir(buildInfo.GetCodeHome()); err != nil {
//d.errappend(ErrorAndSolve(err, "清理cache dir错误", "请提交代码到仓库"))
return d.errors
}
}
@ -186,14 +196,56 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
return d.errors
}
logrus.Errorf("svn checkout or update error,%s", err.Error())
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("获取代码失败"), "请确认仓库能否正常访问,或查看社区文档"))
d.errappend(ErrorAndSolve(FatalError, "获取代码失败"+err.Error(), "请确认仓库能否正常访问,或查看社区文档"))
return d.errors
}
//get branchs
d.branchs = rs.Branchs
return nil
}
logrus.Debugf("start get service code by %s server type", csi.ServerType)
ossFunc := func() ParseErrorList {
g := got.NewWithContext(context.Background())
util.CheckAndCreateDir(buildInfo.GetCodeHome())
fileName := path.Join(buildInfo.GetCodeHome(), path.Base(csi.RepositoryURL))
if err := g.Do(&got.Download{
URL: csi.RepositoryURL,
Dest: fileName,
Header: []got.GotHeader{
{Key: "Authorization", Value: "Basic " + basicAuth(csi.User, csi.Password)},
},
}); err != nil {
logrus.Errorf("download package file from oss failure %s", err.Error())
d.errappend(ErrorAndSolve(FatalError, "文件下载失败:"+err.Error(), "请确认该文件可以被正常下载"))
return d.errors
}
fi, err := os.Stat(fileName)
if err != nil {
d.errappend(ErrorAndSolve(FatalError, "文件下载失败:"+err.Error(), "请确认该文件可以被正常下载"))
return d.errors
}
logrus.Infof("download package file success, size %d MB", fi.Size()/1024/1024)
ext := path.Ext(csi.RepositoryURL)
switch ext {
case ".tar":
if err := util.UnTar(fileName, buildInfo.GetCodeHome(), false); err != nil {
logrus.Errorf("untar package file failure %s", err.Error())
d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为tar规范文件"))
}
case ".tgz", ".tar.gz":
if err := util.UnTar(fileName, buildInfo.GetCodeHome(), true); err != nil {
logrus.Errorf("untar package file failure %s", err.Error())
d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为tgz规范文件"))
}
case ".zip":
if err := util.Unzip(fileName, buildInfo.GetCodeHome()); err != nil {
logrus.Errorf("untar package file failure %s", err.Error())
d.errappend(ErrorAndSolve(FatalError, "文件解压失败", "请确认该文件是否为zip规范文件"))
}
}
logrus.Infof("unpack package file success")
return d.errors
}
logrus.Debugf("start get service %s code by %s server type", csi.ServiceID, csi.ServerType)
//获取代码仓库
switch csi.ServerType {
case "git":
@ -204,6 +256,10 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
if err := svnFunc(); err != nil && err.IsFatalError() {
return err
}
case "oss":
if err := ossFunc(); err != nil && err.IsFatalError() {
return err
}
default:
//default git
logrus.Warningf("do not get void server type,default use git")
@ -211,14 +267,6 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
return err
}
}
// The source code is useless after the test is completed, and needs to be deleted.
defer func() {
if sources.CheckFileExist(buildInfo.GetCodeHome()) {
if err := sources.RemoveDir(buildInfo.GetCodeHome()); err != nil {
logrus.Warningf("remove source code: %v", err)
}
}
}()
//read rainbondfile
rbdfileConfig, err := code.ReadRainbondFile(buildInfo.GetCodeBuildAbsPath())
@ -303,7 +351,7 @@ func (d *SourceCodeParse) Parse() ParseErrorList {
d.errappend(ErrorAndSolve(FatalError, fmt.Sprintf("error listing modules: %v", err), "check source code for multi-modules"))
return d.errors
}
if services != nil && len(services) > 1 {
if len(services) > 1 {
d.isMulti = true
d.services = services
}
@ -579,3 +627,8 @@ func (d *SourceCodeParse) parseDockerfileInfo(dockerfile string) bool {
d.args = []string{}
return true
}
func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}

View File

@ -70,3 +70,25 @@ func TestSourceCode(t *testing.T) {
body, _ := json.Marshal(re)
fmt.Printf("%s \n", string(body))
}
func TestOSSCheck(t *testing.T) {
sc := sources.CodeSourceInfo{
ServerType: "oss",
RepositoryURL: "http://8081.gr021644.64q1jlfb.17f4cc.grapps.cn/artifactory/dev/java-war-demo-master.tar.gz",
User: "demo",
Password: "gr123465!",
}
b, _ := json.Marshal(sc)
p := CreateSourceCodeParse(string(b), nil)
err := p.Parse()
if err != nil && err.IsFatalError() {
t.Fatal(err)
}
re := ServiceCheckResult{
CheckStatus: "Success",
ErrorInfos: err,
ServiceInfo: p.GetServiceInfo(),
}
body, _ := json.Marshal(re)
fmt.Printf("%s \n", string(body))
}

2
go.mod
View File

@ -58,6 +58,7 @@ require (
github.com/kr/pretty v0.2.1 // indirect
github.com/kr/pty v1.1.8
github.com/mattn/go-runewidth v0.0.6
github.com/melbahja/got v0.5.0
github.com/mitchellh/go-ps v1.0.0
github.com/mitchellh/go-wordwrap v1.0.0
github.com/mitchellh/mapstructure v1.3.3
@ -99,6 +100,7 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e // indirect
google.golang.org/grpc v1.33.2
google.golang.org/protobuf v1.25.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/src-d/go-git.v4 v4.13.1
gopkg.in/yaml.v2 v2.4.0

20
go.sum
View File

@ -93,12 +93,6 @@ github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/GLYASAI/rainbond-oam v0.0.0-20210720142412-6faa7418c0ab h1:IvNXJNJhmZeMPbulJ7SMtcR9DduQIj0FyUsS5dcLqAQ=
github.com/GLYASAI/rainbond-oam v0.0.0-20210720142412-6faa7418c0ab/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
github.com/GLYASAI/rainbond-oam v0.0.0-20210720150350-96daaa6aec86 h1:wCxXRbtOrsCjw9+gp8n5t1kTFy9/xsWyuZOPR5zvypY=
github.com/GLYASAI/rainbond-oam v0.0.0-20210720150350-96daaa6aec86/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
github.com/GLYASAI/rainbond-oam v0.0.0-20210721020036-158e1be667dc h1:UMR1n8LoCY79PvroSDDlGfSSomqEgeQAgZ+k8MBgcYk=
github.com/GLYASAI/rainbond-oam v0.0.0-20210721020036-158e1be667dc/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU=
@ -384,6 +378,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k=
github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0=
@ -699,10 +694,6 @@ github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkY
github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
github.com/goodrain/gorm-bulk-upsert v1.0.1-0.20210608013724-7e7870d16357 h1:kdSSrpA5yNvgqbfpMlEr8bvQWiLc1Uz9g0vYf3JVT7s=
github.com/goodrain/gorm-bulk-upsert v1.0.1-0.20210608013724-7e7870d16357/go.mod h1:b7/GgeVNbf/SFw4FYIWslxNV5I10C9Mhf/++3jsDk3M=
github.com/goodrain/rainbond-oam v0.0.0-20210206075623-511d0796af43 h1:xhUwEWQKk+maL6CmV5Y6kxKb+jA/RvN6SZcDbNm51FM=
github.com/goodrain/rainbond-oam v0.0.0-20210206075623-511d0796af43/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
github.com/goodrain/rainbond-oam v0.0.0-20210531110322-0fc33198e2d1 h1:XnipcyrdVCxtKYpy7APoaQwXOX4WGwxgZVpy1XvYb/A=
github.com/goodrain/rainbond-oam v0.0.0-20210531110322-0fc33198e2d1/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
github.com/goodrain/rainbond-oam v0.0.0-20210721020036-158e1be667dc h1:hCtxb/Yy4G+wEc2n+yaXx3j4SF/s34zNI8XK5qkHqXk=
github.com/goodrain/rainbond-oam v0.0.0-20210721020036-158e1be667dc/go.mod h1:/dRehR3e1pGexOaIDjA44AHBlVPbb7v+O7GWAVyo740=
github.com/goodrain/rainbond-operator v1.3.1-0.20210401055914-f8fe4bf89a21 h1:iCPI96slvJv88iPc1NJW8zhpkiza0kwB0jtsuZIJLRQ=
@ -1081,6 +1072,8 @@ github.com/mdlayher/netlink v1.1.0 h1:mpdLgm+brq10nI9zM1BpX1kpDbh3NLl3RSnVq6ZSkf
github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee h1:hZDujBrW3ye2xxdKNFYT59D4yCH5Q0zLuNBNtysKtok=
github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee/go.mod h1:Evt/EIne46u9PtQbeTx2NTcqURpr5K4SvKtGmBuDPN8=
github.com/melbahja/got v0.5.0 h1:tFp5WasOUeZ/SiW82G4SS8clb5Q/Pw6IFYvOcLgt/ZY=
github.com/melbahja/got v0.5.0/go.mod h1:z/sG672dxMFNDZtZ9CHBbfuYeTzV0Xqilq6BUEw41oY=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@ -1499,6 +1492,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo=
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
@ -1545,6 +1540,11 @@ github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUA
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
gitlab.com/poldi1405/go-ansi v1.1.0 h1:BrWoD9TivJKjdnQUwiHVzyqAemDiRpDVBCbbNOPoySQ=
gitlab.com/poldi1405/go-ansi v1.1.0/go.mod h1:TLoRttGdPaq5H2qfF7I1cC5Lt2WXtQeIkFurjbUf6OI=
gitlab.com/poldi1405/go-indicators v0.0.0-20200820134929-9b373aa411a5/go.mod h1:jn34qwBiXTHz73wD9neAMeJGCeMijVro+WCqk1RJXL4=
gitlab.com/poldi1405/go-indicators v1.0.0 h1:miZ7CisCvAH6G16OuY6ATtC8QUUjVCYabNgVsRYjJIs=
gitlab.com/poldi1405/go-indicators v1.0.0/go.mod h1:jn34qwBiXTHz73wD9neAMeJGCeMijVro+WCqk1RJXL4=
go.elastic.co/apm v1.5.0/go.mod h1:OdB9sPtM6Vt7oz3VXt7+KR96i9li74qrxBGHTQygFvk=
go.elastic.co/apm/module/apmhttp v1.5.0/go.mod h1:1FbmNuyD3ddauwzgVwFB0fqY6KbZt3JkV187tGCYYhY=
go.elastic.co/apm/module/apmot v1.5.0/go.mod h1:d2KYwhJParTpyw2WnTNy8geNlHKKFX+4oK3YLlsesWE=

View File

@ -527,6 +527,16 @@ func Zip(source, target string) error {
return err
}
func UnTar(archive, target string, zip bool) error {
parameter := "-x"
if zip {
parameter = "-zx"
}
command := []string{"tar", parameter, "-C", target, "-f", archive}
cmd := exec.Command(command[0], command[1:]...)
return cmd.Run()
}
//Unzip archive file to target dir
func Unzip(archive, target string) error {
reader, err := zip.OpenDirectReader(archive)