perf: public build cache (#1745)

Co-authored-by: 张启航 <zhangqh@goodrain.com>

builder/build/build.go

@@ -81,36 +81,37 @@ type Response struct {
 
 //Request build input
 type Request struct {
-	BuildKitImage string
-	BuildKitArgs  []string
-	BuildKitCache bool
-	RbdNamespace  string
-	GRDataPVCName string
-	CachePVCName  string
-	CacheMode     string
-	CachePath     string
-	TenantID      string
-	SourceDir     string
-	CacheDir      string
-	TGZDir        string
-	RepositoryURL string
-	CodeSouceInfo sources.CodeSourceInfo
-	Branch        string
-	ServiceAlias  string
-	ServiceID     string
-	DeployVersion string
-	Runtime       string
-	ServerType    string
-	Commit        Commit
-	Lang          code.Lang
-	BuildEnvs     map[string]string
-	Logger        event.Logger
-	ImageClient   sources.ImageClient
-	KubeClient    kubernetes.Interface
-	ExtraHosts    []string
-	HostAlias     []HostAlias
-	Ctx           context.Context
-	Arch          string
+	BuildKitImage    string
+	BuildKitArgs     []string
+	BuildKitCache    bool
+	BuildSharedCache bool
+	RbdNamespace     string
+	GRDataPVCName    string
+	CachePVCName     string
+	CacheMode        string
+	CachePath        string
+	TenantID         string
+	SourceDir        string
+	CacheDir         string
+	TGZDir           string
+	RepositoryURL    string
+	CodeSouceInfo    sources.CodeSourceInfo
+	Branch           string
+	ServiceAlias     string
+	ServiceID        string
+	DeployVersion    string
+	Runtime          string
+	ServerType       string
+	Commit           Commit
+	Lang             code.Lang
+	BuildEnvs        map[string]string
+	Logger           event.Logger
+	ImageClient      sources.ImageClient
+	KubeClient       kubernetes.Interface
+	ExtraHosts       []string
+	HostAlias        []HostAlias
+	Ctx              context.Context
+	Arch             string
 }
 
 // HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the

builder/build/code_build.go

@@ -184,20 +184,19 @@ func (s *slugBuild) stopPreBuildJob(re *Request) error {
 	return nil
 }
 
-func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string) (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) {
+func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string, buildNoCache bool) (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) {
 	slugSubPath := strings.TrimPrefix(re.TGZDir, "/grdata/")
 	lazyloading := sourceTarFileName == ""
 	sourceTarPath := strings.TrimPrefix(sourceTarFileName, "/cache/")
 	cacheSubPath := strings.TrimPrefix(re.CacheDir, "/cache/")
+	if s.re.BuildSharedCache {
+		cacheSubPath = path.Join("build/cache", re.Lang.String())
+	}
 
 	hostPathType := corev1.HostPathDirectoryOrCreate
 	unset := corev1.HostPathUnset
 	if re.CacheMode == "hostpath" {
 		volumeMounts = []corev1.VolumeMount{
-			{
-				Name:      "cache",
-				MountPath: "/tmp/cache",
-			},
 			{
 				Name:      "slug",
 				MountPath: "/tmp/slug",
@@ -219,15 +218,6 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
 					},
 				},
 			},
-			{
-				Name: "cache",
-				VolumeSource: corev1.VolumeSource{
-					HostPath: &corev1.HostPathVolumeSource{
-						Path: path.Join(re.CachePath, cacheSubPath),
-						Type: &hostPathType,
-					},
-				},
-			},
 		}
 		if !lazyloading {
 			volumes = append(volumes, corev1.Volume{
@@ -241,6 +231,21 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
 				},
 			})
 		}
+		if !buildNoCache {
+			volumes = append(volumes, corev1.Volume{
+				Name: "cache",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: path.Join(re.CachePath, cacheSubPath),
+						Type: &hostPathType,
+					},
+				},
+			})
+			volumeMounts = append(volumeMounts, corev1.VolumeMount{
+				Name:      "cache",
+				MountPath: "/tmp/cache",
+			})
+		}
 	} else {
 		volumes = []corev1.Volume{
 			{
@@ -261,11 +266,6 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
 			},
 		}
 		volumeMounts = []corev1.VolumeMount{
-			{
-				Name:      "app",
-				MountPath: "/tmp/cache",
-				SubPath:   cacheSubPath,
-			},
 			{
 				Name:      "slug",
 				MountPath: "/tmp/slug",
@@ -279,6 +279,13 @@ func (s *slugBuild) createVolumeAndMount(re *Request, sourceTarFileName string)
 				SubPath:   sourceTarPath,
 			})
 		}
+		if !buildNoCache {
+			volumeMounts = append(volumeMounts, corev1.VolumeMount{
+				Name:      "app",
+				MountPath: "/tmp/cache",
+				SubPath:   cacheSubPath,
+			})
+		}
 	}
 	if re.ServerType == "pkg" {
 		volumeMounts = append(volumeMounts, corev1.VolumeMount{
@@ -341,6 +348,7 @@ func (s *slugBuild) runBuildJob(re *Request) error {
 		envs = append(envs, corev1.EnvVar{Name: "PACKAGE_DOWNLOAD_PASS", Value: re.CodeSouceInfo.Password})
 	}
 	var mavenSettingName string
+	var buildNoCache bool
 	for k, v := range re.BuildEnvs {
 		if k == "MAVEN_SETTING_NAME" {
 			mavenSettingName = v
@@ -362,6 +370,9 @@ func (s *slugBuild) runBuildJob(re *Request) error {
 				}
 			}
 		}
+		if k == "NO_CACHE" && v == "True" {
+			buildNoCache = true
+		}
 	}
 	podSpec := corev1.PodSpec{
 		RestartPolicy: corev1.RestartPolicyOnFailure,
@@ -400,7 +411,7 @@ func (s *slugBuild) runBuildJob(re *Request) error {
 	}
 	logrus.Debugf("request is: %+v", re)
 
-	volumes, mounts := s.createVolumeAndMount(re, sourceTarFileName)
+	volumes, mounts := s.createVolumeAndMount(re, sourceTarFileName, buildNoCache)
 	podSpec.Volumes = volumes
 	container := corev1.Container{
 		Name:      name,

builder/exector/build_from_sourcecode_run.go

@@ -46,39 +46,40 @@ import (
 
 //SourceCodeBuildItem SouceCodeBuildItem
 type SourceCodeBuildItem struct {
-	Namespace     string       `json:"namespace"`
-	TenantName    string       `json:"tenant_name"`
-	GRDataPVCName string       `json:"gr_data_pvc_name"`
-	CachePVCName  string       `json:"cache_pvc_name"`
-	CacheMode     string       `json:"cache_mode"`
-	CachePath     string       `json:"cache_path"`
-	ServiceAlias  string       `json:"service_alias"`
-	Action        string       `json:"action"`
-	Arch          string       `json:"arch"`
-	DestImage     string       `json:"dest_image"`
-	Logger        event.Logger `json:"logger"`
-	EventID       string       `json:"event_id"`
-	CacheDir      string       `json:"cache_dir"`
-	TGZDir        string       `json:"tgz_dir"`
-	ImageClient   sources.ImageClient
-	BuildKitImage string
-	BuildKitArgs  []string
-	BuildKitCache bool
-	KubeClient    kubernetes.Interface
-	RbdNamespace  string
-	RbdRepoName   string
-	TenantID      string
-	ServiceID     string
-	DeployVersion string
-	Lang          string
-	Runtime       string
-	BuildEnvs     map[string]string
-	CodeSouceInfo sources.CodeSourceInfo
-	RepoInfo      *sources.RepostoryBuildInfo
-	commit        Commit
-	Configs       map[string]gjson.Result `json:"configs"`
-	Ctx           context.Context
-	FailCause     string
+	Namespace        string       `json:"namespace"`
+	TenantName       string       `json:"tenant_name"`
+	GRDataPVCName    string       `json:"gr_data_pvc_name"`
+	CachePVCName     string       `json:"cache_pvc_name"`
+	CacheMode        string       `json:"cache_mode"`
+	CachePath        string       `json:"cache_path"`
+	ServiceAlias     string       `json:"service_alias"`
+	Action           string       `json:"action"`
+	Arch             string       `json:"arch"`
+	DestImage        string       `json:"dest_image"`
+	Logger           event.Logger `json:"logger"`
+	EventID          string       `json:"event_id"`
+	CacheDir         string       `json:"cache_dir"`
+	TGZDir           string       `json:"tgz_dir"`
+	ImageClient      sources.ImageClient
+	BuildKitImage    string
+	BuildKitArgs     []string
+	BuildKitCache    bool
+	BuildSharedCache bool
+	KubeClient       kubernetes.Interface
+	RbdNamespace     string
+	RbdRepoName      string
+	TenantID         string
+	ServiceID        string
+	DeployVersion    string
+	Lang             string
+	Runtime          string
+	BuildEnvs        map[string]string
+	CodeSouceInfo    sources.CodeSourceInfo
+	RepoInfo         *sources.RepostoryBuildInfo
+	commit           Commit
+	Configs          map[string]gjson.Result `json:"configs"`
+	Ctx              context.Context
+	FailCause        string
 }
 
 //Commit code Commit
@@ -327,35 +328,36 @@ func (i *SourceCodeBuildItem) codeBuild() (*build.Response, error) {
 		return nil, err
 	}
 	buildReq := &build.Request{
-		BuildKitImage: i.BuildKitImage,
-		BuildKitArgs:  i.BuildKitArgs,
-		BuildKitCache: i.BuildKitCache,
-		RbdNamespace:  i.RbdNamespace,
-		SourceDir:     i.RepoInfo.GetCodeBuildAbsPath(),
-		CacheDir:      i.CacheDir,
-		TGZDir:        i.TGZDir,
-		RepositoryURL: i.RepoInfo.RepostoryURL,
-		CodeSouceInfo: i.CodeSouceInfo,
-		ServiceAlias:  i.ServiceAlias,
-		ServiceID:     i.ServiceID,
-		TenantID:      i.TenantID,
-		ServerType:    i.CodeSouceInfo.ServerType,
-		Runtime:       i.Runtime,
-		Branch:        i.CodeSouceInfo.Branch,
-		DeployVersion: i.DeployVersion,
-		Commit:        build.Commit{User: i.commit.Author, Message: i.commit.Message, Hash: i.commit.Hash},
-		Lang:          code.Lang(i.Lang),
-		BuildEnvs:     i.BuildEnvs,
-		Logger:        i.Logger,
-		ImageClient:   i.ImageClient,
-		KubeClient:    i.KubeClient,
-		HostAlias:     hostAlias,
-		Ctx:           i.Ctx,
-		GRDataPVCName: i.GRDataPVCName,
-		CachePVCName:  i.CachePVCName,
-		CacheMode:     i.CacheMode,
-		CachePath:     i.CachePath,
-		Arch:          i.Arch,
+		BuildKitImage:    i.BuildKitImage,
+		BuildKitArgs:     i.BuildKitArgs,
+		BuildKitCache:    i.BuildKitCache,
+		BuildSharedCache: i.BuildSharedCache,
+		RbdNamespace:     i.RbdNamespace,
+		SourceDir:        i.RepoInfo.GetCodeBuildAbsPath(),
+		CacheDir:         i.CacheDir,
+		TGZDir:           i.TGZDir,
+		RepositoryURL:    i.RepoInfo.RepostoryURL,
+		CodeSouceInfo:    i.CodeSouceInfo,
+		ServiceAlias:     i.ServiceAlias,
+		ServiceID:        i.ServiceID,
+		TenantID:         i.TenantID,
+		ServerType:       i.CodeSouceInfo.ServerType,
+		Runtime:          i.Runtime,
+		Branch:           i.CodeSouceInfo.Branch,
+		DeployVersion:    i.DeployVersion,
+		Commit:           build.Commit{User: i.commit.Author, Message: i.commit.Message, Hash: i.commit.Hash},
+		Lang:             code.Lang(i.Lang),
+		BuildEnvs:        i.BuildEnvs,
+		Logger:           i.Logger,
+		ImageClient:      i.ImageClient,
+		KubeClient:       i.KubeClient,
+		HostAlias:        hostAlias,
+		Ctx:              i.Ctx,
+		GRDataPVCName:    i.GRDataPVCName,
+		CachePVCName:     i.CachePVCName,
+		CacheMode:        i.CacheMode,
+		CachePath:        i.CachePath,
+		Arch:             i.Arch,
 	}
 	res, err := codeBuild.Build(buildReq)
 	return res, err

builder/exector/exector.go

@@ -123,6 +123,7 @@ func NewManager(conf option.Config, mqc mqclient.MQClient) (Manager, error) {
 		BuildKitImage:     conf.BuildKitImage,
 		BuildKitArgs:      strings.Split(conf.BuildKitArgs, "&"),
 		BuildKitCache:     conf.BuildKitCache,
+		BuildSharedCache:  conf.BuildSharedCache,
 		KubeClient:        kubeClient,
 		EtcdCli:           etcdCli,
 		mqClient:          mqc,
@@ -139,6 +140,7 @@ type exectorManager struct {
 	BuildKitImage     string
 	BuildKitArgs      []string
 	BuildKitCache     bool
+	BuildSharedCache  bool
 	KubeClient        kubernetes.Interface
 	EtcdCli           *clientv3.Client
 	tasks             chan *pb.TaskMessage
@@ -351,6 +353,7 @@ func (e *exectorManager) buildFromSourceCode(task *pb.TaskMessage) {
 	i.BuildKitImage = e.BuildKitImage
 	i.BuildKitArgs = e.BuildKitArgs
 	i.BuildKitCache = e.BuildKitCache
+	i.BuildSharedCache = e.BuildSharedCache
 	i.KubeClient = e.KubeClient
 	i.RbdNamespace = e.cfg.RbdNamespace
 	i.RbdRepoName = e.cfg.RbdRepoName

cmd/builder/option/option.go

@@ -39,6 +39,7 @@ type Config struct {
 	MysqlConnectionInfo  string
 	BuildKitImage        string
 	BuildKitArgs         string
+	BuildSharedCache     bool
 	BuildKitCache        bool
 	DBType               string
 	PrometheusMetricPath string
@@ -108,6 +109,7 @@ func (a *Builder) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&a.RuntimeEndpoint, "runtime-endpoint", sources.RuntimeEndpointContainerd, "container runtime endpoint")
 	fs.StringVar(&a.BuildKitArgs, "buildkit-args", "", "buildkit build image container args config,need '&' split")
 	fs.BoolVar(&a.BuildKitCache, "buildkit-cache", true, "whether to enable the buildkit image cache")
+	fs.BoolVar(&a.BuildSharedCache, "build-shared-cache", true, "build shared cache")
 }
 
 //SetLog 设置log