Merge branch 'master' into 961

api/controller/gateway.go

@@ -312,6 +312,7 @@ func (g *GatewayStruct) RuleConfig(w http.ResponseWriter, r *http.Request) {
 	if !ok {
 		return
 	}
+
 	sid := r.Context().Value(middleware.ContextKey("service_id")).(string)
 	eventID := r.Context().Value(middleware.ContextKey("event_id")).(string)
 	req.ServiceID = sid

api/model/gateway_model.go

@@ -129,8 +129,8 @@ type Body struct {
 	ProxyBodySize       int          `json:"proxy_body_size,omitempty" validate:"proxy_body_size|required"`
 	SetHeaders          []*SetHeader `json:"set_headers,omitempty" `
 	Rewrites            []*Rewrite   `json:"rewrite,omitempty"`
-	ProxyBufferSize     int          `json:"proxy_buffer_size,omitempty"`
+	ProxyBufferSize     int          `json:"proxy_buffer_size,omitempty" validate:"proxy_buffer_size|numeric_between:1,65535"`
-	ProxyBufferNumbers  int          `json:"proxy_buffer_numbers,omitempty"`
+	ProxyBufferNumbers  int          `json:"proxy_buffer_numbers,omitempty" validate:"proxy_buffer_size|numeric_between:1,65535"`
 }
 
 //SetHeader set header

builder/job/job.go

@@ -236,6 +236,7 @@ func (c *controller) GetLanguageBuildSetting(ctx context.Context, lang code.Lang
 	config, err := c.KubeClient.CoreV1().ConfigMaps(c.namespace).Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
 		logrus.Errorf("get configmap %s failure  %s", name, err.Error())
+		return ""
 	}
 	if config != nil {
 		return name

builder/sources/registry/errors.go

@@ -23,4 +23,11 @@ import "github.com/pkg/errors"
 // registry error
 var (
 	ErrRegistryNotFound = errors.New("registry not found")
+  
+	// ErrRepositoryNotFound means the repository can not be found.
+	ErrRepositoryNotFound = errors.New("repository not found")
+
+	ErrManifestNotFound = errors.New("manifest not found")
+
+	ErrOperationIsUnsupported = errors.New("The operation is unsupported")
 )

builder/sources/registry/manifest.go

@@ -23,10 +23,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strings"
 
 	manifestV1 "github.com/docker/distribution/manifest/schema1"
 	manifestV2 "github.com/docker/distribution/manifest/schema2"
 	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -119,6 +121,42 @@ func (registry *Registry) ManifestDigest(repository, reference string) (digest.D
 	return digest.Parse(resp.Header.Get("Docker-Content-Digest"))
 }
 
+// ManifestDigestV2 -
+func (registry *Registry) ManifestDigestV2(repository, reference string) (digest.Digest, error) {
+	url := registry.url("/v2/%s/manifests/%s", repository, reference)
+	registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)
+
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Accept", manifestV2.MediaTypeManifest)
+
+	resp, err := registry.Client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("do request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 404 {
+		return "", errors.Wrap(ErrManifestNotFound, "get digest v2")
+	}
+
+	if resp.StatusCode != 200 {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			logrus.Warningf("read digest v2 body")
+		}
+		msg := fmt.Sprintf("unexpect status code: %d", resp.StatusCode)
+		if len(body) > 0 {
+			msg += "; " + string(body)
+		}
+		return "", errors.New(msg)
+	}
+
+	return digest.Parse(resp.Header.Get("Docker-Content-Digest"))
+}
+
 // DeleteManifest -
 func (registry *Registry) DeleteManifest(repository string, digest digest.Digest) error {
 	url := registry.url("/v2/%s/manifests/%s", repository, digest)
@@ -129,12 +167,15 @@ func (registry *Registry) DeleteManifest(repository string, digest digest.Digest
 		return err
 	}
 	resp, err := registry.Client.Do(req)
+	if err != nil {
+		if strings.Contains(err.Error(), "The operation is unsupported.") {
+			return errors.Wrap(ErrOperationIsUnsupported, "delete manifest")
+		}
+		return errors.Wrap(err, "do request")
+	}
 	if resp != nil {
 		defer resp.Body.Close()
 	}
-	if err != nil {
-		return err
-	}
 	return nil
 }
 

builder/sources/registry/tag.go

@@ -18,10 +18,17 @@
 
 package registry
 
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
 type tagsResponse struct {
 	Tags []string `json:"tags"`
 }
 
+// Tags returns the list of tags fo the repository.
 func (registry *Registry) Tags(repository string) (tags []string, err error) {
 	url := registry.url("/v2/%s/tags/list", repository)
 
@@ -37,6 +44,9 @@ func (registry *Registry) Tags(repository string) (tags []string, err error) {
 			tags = append(tags, response.Tags...)
 			continue
 		default:
+			if strings.Contains(err.Error(), "404") {
+				return nil, errors.Wrap(ErrRepositoryNotFound, "list tags")
+			}
 			return nil, err
 		}
 	}

db/dao/dao.go

@@ -152,6 +152,7 @@ type TenantServiceDeleteDao interface {
 	Dao
 	GetTenantServicesDeleteByCreateTime(createTime time.Time) ([]*model.TenantServicesDelete, error)
 	DeleteTenantServicesDelete(record *model.TenantServicesDelete) error
+	List() ([]*model.TenantServicesDelete, error)
 }
 
 //TenantServicesPortDao TenantServicesPortDao
@@ -403,6 +404,7 @@ type VersionInfoDao interface {
 	DeleteVersionInfo(obj *model.VersionInfo) error
 	DeleteFailureVersionInfo(timePoint time.Time, status string, serviceIDList []string) error
 	SearchVersionInfo() ([]*model.VersionInfo, error)
+	ListByServiceIDStatus(serviceID string, finalStatus *bool) ([]*model.VersionInfo, error)
 }
 
 //RegionUserInfoDao UserRegionInfoDao

db/mysql/dao/tenants.go

@@ -30,6 +30,7 @@ import (
 	"github.com/goodrain/rainbond/db/errors"
 	"github.com/goodrain/rainbond/db/model"
 	"github.com/jinzhu/gorm"
+	pkgerr "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -594,6 +595,15 @@ func (t *TenantServicesDeleteImpl) DeleteTenantServicesDelete(record *model.Tena
 	return nil
 }
 
+// List returns a list of TenantServicesDeletes.
+func (t *TenantServicesDeleteImpl) List() ([]*model.TenantServicesDelete, error) {
+	var components []*model.TenantServicesDelete
+	if err := t.DB.Find(&components).Error; err != nil {
+		return nil, pkgerr.Wrap(err, "list deleted components")
+	}
+	return components, nil
+}
+
 //TenantServicesPortDaoImpl 租户应用端口操作
 type TenantServicesPortDaoImpl struct {
 	DB *gorm.DB

db/mysql/dao/version.go

@@ -19,12 +19,12 @@
 package dao
 
 import (
-	"github.com/goodrain/rainbond/db/errors"
-	"github.com/goodrain/rainbond/db/model"
-
 	"time"
 
+	"github.com/goodrain/rainbond/db/errors"
+	"github.com/goodrain/rainbond/db/model"
 	"github.com/jinzhu/gorm"
+	pkgerr "github.com/pkg/errors"
 )
 
 //DeleteVersionByEventID DeleteVersionByEventID
@@ -90,6 +90,19 @@ func (c *VersionInfoDaoImpl) ListSuccessfulOnes() ([]*model.VersionInfo, error)
 	return versoins, nil
 }
 
+// ListByServiceIDStatus returns a list of versions based on the given serviceID and finalStatus.
+func (c *VersionInfoDaoImpl) ListByServiceIDStatus(serviceID string, finalStatus *bool) ([]*model.VersionInfo, error) {
+	db := c.DB.Where("service_id=?", serviceID)
+	if finalStatus != nil {
+		db = db.Where("final_status=?", "success")
+	}
+	var versoins []*model.VersionInfo
+	if err := db.Find(&versoins).Error; err != nil {
+		return nil, pkgerr.Wrap(err, "list versions")
+	}
+	return versoins, nil
+}
+
 //GetVersionByEventID get version by event id
 func (c *VersionInfoDaoImpl) GetVersionByEventID(eventID string) (*model.VersionInfo, error) {
 	var result model.VersionInfo

gateway/annotations/proxy/proxy.go

@@ -18,16 +18,15 @@ package proxy
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 
-	"golang.org/x/net/http/httpguts"
-
-	"github.com/goodrain/rainbond/gateway/controller/config"
-	"github.com/sirupsen/logrus"
-	extensions "k8s.io/api/extensions/v1beta1"
-
 	"github.com/goodrain/rainbond/gateway/annotations/parser"
 	"github.com/goodrain/rainbond/gateway/annotations/resolver"
+	"github.com/goodrain/rainbond/gateway/controller/config"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/net/http/httpguts"
+	extensions "k8s.io/api/extensions/v1beta1"
 )
 
 // Config returns the proxy timeout to use in the upstream server/s
@@ -61,16 +60,20 @@ func (s *Config) Validation() error {
 			return fmt.Errorf("header %s value %s is valid", k, v)
 		}
 	}
-	if s.ProxyBuffering == "" {
+	if !s.validateBuffering(s.ProxyBuffering) {
+		logrus.Warningf("invalid proxy buffering: %s; use the default one: %s", s.ProxyBuffering, defBackend.ProxyBuffering)
 		s.ProxyBuffering = defBackend.ProxyBuffering
 	}
-	if s.BufferSize == "" {
+	if !s.validateBufferSize() {
+		logrus.Warningf("invalid proxy buffer size: %s; use the default one: %s", s.BufferSize, defBackend.ProxyBufferSize)
 		s.BufferSize = defBackend.ProxyBufferSize
 	}
-	if s.BuffersNumber == 0 {
+	if s.BuffersNumber <= 0 {
+		logrus.Warningf("invalid buffer number: %d; use the default one: %d", s.BuffersNumber, defBackend.ProxyBuffersNumber)
 		s.BuffersNumber = defBackend.ProxyBuffersNumber
 	}
-	if s.RequestBuffering == "" {
+	if s.validateBuffering(s.RequestBuffering) {
+		logrus.Warningf("invalid reqeust buffering: %s; use the default one: %s", s.RequestBuffering, defBackend.ProxyRequestBuffering)
 		s.RequestBuffering = defBackend.ProxyRequestBuffering
 	}
 	if s.CookieDomain == "" {
@@ -82,6 +85,15 @@ func (s *Config) Validation() error {
 	return nil
 }
 
+func (s *Config) validateBufferSize() bool {
+	reg := regexp.MustCompile(`^[1-9]\d*k$`)
+	return reg.MatchString(s.BufferSize)
+}
+
+func (s *Config) validateBuffering(buffering string) bool {
+	return buffering == "off" || buffering == "on"
+}
+
 //NewProxyConfig new proxy config
 func NewProxyConfig() Config {
 	defBackend := config.NewDefault()

go.mod

@@ -101,7 +101,7 @@ require (
 	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
 	golang.org/x/sys v0.0.0-20201223074533-0d417f636930
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
-	golang.org/x/tools v0.0.0-20201228162255-34cd474b9958 // indirect
+	golang.org/x/tools v0.0.0-20201228162255-34cd474b9958
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e // indirect
 	google.golang.org/grpc v1.33.2

grctl/cmd/cmd.go

@@ -43,6 +43,7 @@ func GetCmds() []cli.Command {
 	cmds = append(cmds, NewCmdGateway())
 	cmds = append(cmds, NewCmdEnvoy())
 	cmds = append(cmds, NewCmdConfig())
+	cmds = append(cmds, NewCmdRegistry())
 	return cmds
 }
 

grctl/cmd/registry.go

@@ -0,0 +1,127 @@
+// Copyright (C) 2014-2021 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+
+	rainbondv1alpha1 "github.com/goodrain/rainbond-operator/api/v1alpha1"
+	"github.com/goodrain/rainbond/db"
+	"github.com/goodrain/rainbond/db/config"
+	"github.com/goodrain/rainbond/grctl/clients"
+	"github.com/goodrain/rainbond/grctl/registry"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// NewCmdRegistry registry cmd
+func NewCmdRegistry() cli.Command {
+	c := cli.Command{
+		Name:  "registry",
+		Usage: "grctl registry [command]",
+		Subcommands: []cli.Command{
+			{
+				Name: "cleanup",
+				Usage: `Clean up free images in the registry.
+	The command 'grctl registry cleanup' will delete the index of free images in registry.
+	Then you have to exec the command below to remove blobs from the filesystem:
+		bin/registry garbage-collect [--dry-run] /path/to/config.yml
+	More Detail: https://docs.docker.com/registry/garbage-collection/#run-garbage-collection.
+				`,
+				Flags: []cli.Flag{
+					cli.StringFlag{
+						Name:   "namespace, ns",
+						Usage:  "rainbond namespace",
+						EnvVar: "RBDNamespace",
+						Value:  "rbd-system",
+					},
+				},
+				Action: func(c *cli.Context) error {
+					Common(c)
+
+					namespace := c.String("namespace")
+					var cluster rainbondv1alpha1.RainbondCluster
+					if err := clients.RainbondKubeClient.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "rainbondcluster"}, &cluster); err != nil {
+						return errors.Wrap(err, "get configuration from rainbond cluster")
+					}
+
+					dsn, err := databaseDSN(&cluster)
+					if err != nil {
+						return errors.Wrap(err, "get database dsn")
+					}
+
+					dbCfg := config.Config{
+						MysqlConnectionInfo: dsn,
+						DBType:              "mysql",
+					}
+					if err := db.CreateManager(dbCfg); err != nil {
+						return errors.Wrap(err, "create database manager")
+					}
+
+					registryConfig := cluster.Spec.ImageHub
+					cleaner, err := registry.NewRegistryCleaner(registryConfig.Domain, registryConfig.Username, registryConfig.Password)
+					if err != nil {
+						return errors.WithMessage(err, "create registry cleaner")
+					}
+
+					cleaner.Cleanup()
+
+					return nil
+				},
+			},
+		},
+	}
+	return c
+}
+
+func databaseDSN(rainbondcluster *rainbondv1alpha1.RainbondCluster) (string, error) {
+	database := rainbondcluster.Spec.RegionDatabase
+	if database != nil {
+		return fmt.Sprintf("%s:%s@tcp(%s)/%s", database.Username, database.Password, database.Host, database.Name), nil
+	}
+	// default name of rbd-db pod is rbd-db-0
+	pod, err := clients.K8SClient.CoreV1().Pods(rainbondcluster.Namespace).Get(context.Background(), "rbd-db-0", metav1.GetOptions{})
+	if err != nil {
+		return "", errors.Wrap(err, "get pod rbd-db-0")
+	}
+	host := pod.Status.PodIP
+	name := "region"
+	for _, ct := range pod.Spec.Containers {
+		if ct.Name != "rbd-db" {
+			continue
+		}
+		for _, env := range ct.Env {
+			if env.Name == "MYSQL_DATABASE" {
+				name = env.Value
+			}
+		}
+	}
+
+	secret, err := clients.K8SClient.CoreV1().Secrets(rainbondcluster.Namespace).Get(context.Background(), "rbd-db", metav1.GetOptions{})
+	if err != nil {
+		return "", errors.Wrap(err, "get secret rbd-db")
+	}
+	username := string(secret.Data["mysql-user"])
+	password := string(secret.Data["mysql-password"])
+
+	return fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", username, password, host, name), nil
+}

grctl/registry/freecomponent.go

@@ -0,0 +1,90 @@
+// Copyright (C) 2014-2021 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package registry
+
+import (
+	"errors"
+
+	"github.com/goodrain/rainbond/builder/sources/registry"
+	"github.com/goodrain/rainbond/db"
+	"github.com/sirupsen/logrus"
+)
+
+var _ FreeImager = &FreeComponent{}
+
+// FreeComponent is resposible for listing the free images belong to free components.
+type FreeComponent struct {
+	reg *registry.Registry
+}
+
+// NewFreeComponent creates a new FreeComponent.
+func NewFreeComponent(reg *registry.Registry) *FreeComponent {
+	return &FreeComponent{
+		reg: reg,
+	}
+}
+
+// List return a list of free images belong to free components.
+func (f *FreeComponent) List() ([]*FreeImage, error) {
+	// list free components
+	components, err := db.GetManager().TenantServiceDeleteDao().List()
+	if err != nil {
+		return nil, err
+	}
+
+	var images []*FreeImage
+	for _, cpt := range components {
+		// component.ServiceID is the repository of image
+		freeImages, err := f.listFreeImages(cpt.ServiceID)
+		if err != nil {
+			logrus.Warningf("list free images: %v", err)
+			continue
+		}
+		images = append(images, freeImages...)
+	}
+
+	return images, nil
+}
+
+func (f *FreeComponent) listFreeImages(repository string) ([]*FreeImage, error) {
+	// list tags, then list digest for every tag
+	tags, err := f.reg.Tags(repository)
+	if err != nil {
+		if errors.Is(err, registry.ErrRepositoryNotFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	var images []*FreeImage
+	for _, tag := range tags {
+		digest, err := f.reg.ManifestDigestV2(repository, tag)
+		if err != nil {
+			logrus.Warningf("get digest for manifest %s/%s: %v", repository, tag, err)
+			continue
+		}
+		images = append(images, &FreeImage{
+			Repository: repository,
+			Digest:     digest.String(),
+			Tag:        tag,
+			Type:       string(FreeImageTypeFreeVersion),
+		})
+	}
+	return images, nil
+}

grctl/registry/freeimage.go

@@ -0,0 +1,61 @@
+// Copyright (C) 2014-2021 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package registry
+
+import (
+	"github.com/goodrain/rainbond/builder/sources/registry"
+)
+
+// FreeImageType is the type of FreeImage
+type FreeImageType string
+
+// FreeImageType -
+var (
+	FreeImageTypeFreeComponent FreeImageType = "FreeComponent"
+	FreeImageTypeFreeVersion   FreeImageType = "FreeVersion"
+)
+
+// FreeImage represents a free image.
+type FreeImage struct {
+	Repository string
+	Digest     string
+	Tag        string
+	Type       string
+}
+
+// Key returns the key of the FreeImaeg.
+func (f *FreeImage) Key() string {
+	return f.Repository + "/" + f.Digest
+}
+
+// FreeImager is resposible for listing the free images.
+type FreeImager interface {
+	List() ([]*FreeImage, error)
+}
+
+// NewFreeImageres creates a list of new FreeImager.
+func NewFreeImageres(reg *registry.Registry) map[string]FreeImager {
+	freeImageres := make(map[string]FreeImager, 2)
+	// there are two kinds of free images:
+	// 1. images belongs to the free components
+	// 2. images belongs to the free component versions.
+	freeImageres["free component"] = NewFreeComponent(reg)
+	freeImageres["free version"] = NewFreeVersion(reg)
+	return freeImageres
+}

grctl/registry/freeversion.go

@@ -0,0 +1,135 @@
+// Copyright (C) 2014-2021 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package registry
+
+import (
+	"errors"
+
+	"github.com/goodrain/rainbond/builder/sources/registry"
+	"github.com/goodrain/rainbond/db"
+	"github.com/goodrain/rainbond/util"
+	"github.com/jinzhu/gorm"
+	"github.com/sirupsen/logrus"
+)
+
+var _ FreeImager = &FreeVersion{}
+
+// FreeVersion is resposible for listing the free images belong to free component versions.
+type FreeVersion struct {
+	reg *registry.Registry
+}
+
+// NewFreeVersion creates a free version.
+func NewFreeVersion(reg *registry.Registry) *FreeVersion {
+	return &FreeVersion{
+		reg: reg,
+	}
+}
+
+// List return a list of free images belong to free component versions.
+func (f *FreeVersion) List() ([]*FreeImage, error) {
+	// list components
+	components, err := db.GetManager().TenantServiceDao().GetAllServicesID()
+	if err != nil {
+		return nil, err
+	}
+
+	var images []*FreeImage
+	for _, cpt := range components {
+		// list free tags
+		freeTags, err := f.listFreeTags(cpt.ServiceID)
+		if err != nil {
+			logrus.Warningf("list free tags for repository %s: %v", cpt.ServiceID, err)
+			continue
+		}
+
+		// component.ServiceID is the repository of image
+		freeImages, err := f.listFreeImages(cpt.ServiceID, freeTags)
+		if err != nil {
+			logrus.Warningf("list digests for repository %s: %v", cpt.ServiceID, err)
+			continue
+		}
+		images = append(images, freeImages...)
+	}
+
+	return images, nil
+}
+
+func (f *FreeVersion) listFreeTags(serviceID string) ([]string, error) {
+	// all tags
+	// serviceID is the repository of image
+	tags, err := f.reg.Tags(serviceID)
+	if err != nil {
+		if errors.Is(err, registry.ErrRepositoryNotFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	// versions being used.
+	versions, err := f.listVersions(serviceID)
+	if err != nil {
+		return nil, err
+	}
+
+	var freeTags []string
+	for _, tag := range tags {
+		_, ok := versions[tag]
+		if !ok {
+			freeTags = append(freeTags, tag)
+		}
+	}
+
+	return freeTags, nil
+}
+
+func (f *FreeVersion) listVersions(serviceID string) (map[string]struct{}, error) {
+	// tags being used
+	rawVersions, err := db.GetManager().VersionInfoDao().ListByServiceIDStatus(serviceID, util.Bool(true))
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return map[string]struct{}{}, nil
+		}
+		return nil, err
+	}
+	// make a map of versions
+	versions := make(map[string]struct{})
+	for _, version := range rawVersions {
+		versions[version.BuildVersion] = struct{}{}
+	}
+	return versions, nil
+}
+
+func (f *FreeVersion) listFreeImages(repository string, tags []string) ([]*FreeImage, error) {
+	var images []*FreeImage
+	for _, tag := range tags {
+		digest, err := f.reg.ManifestDigestV2(repository, tag)
+		if err != nil {
+			logrus.Warningf("get digest for manifest %s/%s: %v", repository, tag, err)
+			continue
+		}
+		images = append(images, &FreeImage{
+			Repository: repository,
+			Digest:     digest.String(),
+			Tag:        tag,
+			Type:       string(FreeImageTypeFreeVersion),
+		})
+	}
+	return images, nil
+}

grctl/registry/registry.go

@@ -0,0 +1,127 @@
+// Copyright (C) 2014-2021 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package registry
+
+import (
+	"github.com/goodrain/rainbond/builder/sources/registry"
+	"github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Cleaner is responsible for cleaning up the free images in registry.
+type Cleaner struct {
+	reg          *registry.Registry
+	freeImageres map[string]FreeImager
+}
+
+// NewRegistryCleaner creates a new Cleaner.
+func NewRegistryCleaner(url, username, password string) (*Cleaner, error) {
+	reg, err := registry.NewInsecure(url, username, password)
+
+	freeImageres := NewFreeImageres(reg)
+
+	return &Cleaner{
+		reg:          reg,
+		freeImageres: freeImageres,
+	}, err
+}
+
+// Cleanup cleans up the free image in the registry.
+func (r *Cleaner) Cleanup() {
+	logrus.Info("Start cleaning up the free images. Please be patient.")
+	logrus.Info("The clean up time will be affected by the number of free images and the network environment.")
+
+	// list images needed to be cleaned up
+	freeImages := r.ListFreeImages()
+	if len(freeImages) == 0 {
+		logrus.Info("Free images not Found")
+		return
+	}
+	logrus.Infof("Found %d free images", len(freeImages))
+
+	// delete images
+	if err := r.DeleteImages(freeImages); err != nil {
+		if errors.Is(err, registry.ErrOperationIsUnsupported) {
+			logrus.Warningf(`The operation image deletion is unsupported.
+You can try to add REGISTRY_STORAGE_DELETE_ENABLED=true when start the registry.
+More detail: https://docs.docker.com/registry/configuration/#list-of-configuration-options.
+				`)
+			return
+		}
+		logrus.Warningf("delete images: %v", err)
+		return
+	}
+
+	logrus.Infof(`you have to exec the command below in the registry container to remove blobs from the filesystem:
+	/bin/registry garbage-collect /etc/docker/registry/config.yml
+More Detail: https://docs.docker.com/registry/garbage-collection/#run-garbage-collection.`)
+}
+
+// ListFreeImages return a list of free images needed to be cleaned up.
+func (r *Cleaner) ListFreeImages() []*FreeImage {
+	var freeImages []*FreeImage
+	for name, freeImager := range r.freeImageres {
+		images, err := freeImager.List()
+		if err != nil {
+			logrus.Warningf("list free images for %s", name)
+		}
+		logrus.Infof("Found %d free images from %s", len(images), name)
+		freeImages = append(freeImages, images...)
+	}
+
+	// deduplicate
+	var result []*FreeImage
+	m := make(map[string]struct{})
+	for _, fi := range freeImages {
+		fi := fi
+		key := fi.Key()
+		_, ok := m[key]
+		if ok {
+			continue
+		}
+		m[key] = struct{}{}
+		result = append(result, fi)
+	}
+
+	return result
+}
+
+// DeleteImages deletes images.
+func (r *Cleaner) DeleteImages(freeImages []*FreeImage) error {
+	for _, image := range freeImages {
+		if err := r.deleteManifest(image.Repository, image.Digest); err != nil {
+			if errors.Is(err, registry.ErrOperationIsUnsupported) {
+				return err
+			}
+			logrus.Warningf("delete manifest %s/%s: %v", image.Repository, image.Digest, err)
+			continue
+		}
+		log := logrus.WithField("Type", image.Type).
+			WithField("Component ID", image.Repository).
+			WithField("Build Version", image.Tag).
+			WithField("Digest", image.Digest)
+		log.Infof("image %s/%s deleted", image.Repository, image.Tag)
+	}
+	return nil
+}
+
+func (r *Cleaner) deleteManifest(repository, dig string) error {
+	return r.reg.DeleteManifest(repository, digest.Digest(dig))
+}