Rainbond/worker/appm/conversion/gateway.go
2019-06-25 22:01:03 +08:00

554 lines
17 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package conversion
import (
"fmt"
"os"
"strings"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/db"
"github.com/goodrain/rainbond/db/model"
"github.com/goodrain/rainbond/event"
"github.com/goodrain/rainbond/gateway/annotations/parser"
v1 "github.com/goodrain/rainbond/worker/appm/types/v1"
corev1 "k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
//createDefaultDomain create default domain
func createDefaultDomain(tenantName, serviceAlias string, servicePort int) string {
exDomain := os.Getenv("EX_DOMAIN")
if exDomain == "" {
return ""
}
if strings.Contains(exDomain, ":") {
exDomain = strings.Split(exDomain, ":")[0]
}
if exDomain[0] == '.' {
exDomain = exDomain[1:]
}
exDomain = strings.TrimSpace(exDomain)
return fmt.Sprintf("%d.%s.%s.%s", servicePort, serviceAlias, tenantName, exDomain)
}
//TenantServiceRegist conv inner and outer service regist
func TenantServiceRegist(as *v1.AppService, dbmanager db.Manager) error {
builder, err := AppServiceBuilder(as.ServiceID, string(as.ServiceType), dbmanager, as)
if err != nil {
logrus.Error("create k8s service builder error.", err.Error())
return err
}
k8s, err := builder.Build()
if err != nil {
logrus.Error("error creating app service: ", err.Error())
return err
}
if k8s == nil {
return nil
}
for _, service := range k8s.Services {
as.SetService(service)
}
for _, ing := range k8s.Ingresses {
as.SetIngress(ing)
}
for _, sec := range k8s.Secrets {
as.SetSecret(sec)
}
return nil
}
//AppServiceBuild has the ability to build k8s service, ingress and secret
type AppServiceBuild struct {
serviceID, eventID string
tenant *model.Tenants
service *model.TenantServices
appService *v1.AppService
replicationType string
dbmanager db.Manager
logger event.Logger
}
//AppServiceBuilder returns a AppServiceBuild
func AppServiceBuilder(serviceID, replicationType string, dbmanager db.Manager, as *v1.AppService) (*AppServiceBuild, error) {
service, err := dbmanager.TenantServiceDao().GetServiceByID(serviceID)
if err != nil {
return nil, fmt.Errorf("find service error. %v", err.Error())
}
if service == nil {
return nil, fmt.Errorf("did not find the TenantService corresponding to ServiceID(%s)", serviceID)
}
tenant, err := dbmanager.TenantDao().GetTenantByUUID(service.TenantID)
if err != nil {
return nil, fmt.Errorf("find tenant error. %v", err.Error())
}
if tenant == nil {
return nil, fmt.Errorf("did not find the Tenant corresponding to ServiceID(%s)", serviceID)
}
return &AppServiceBuild{
serviceID: serviceID,
dbmanager: dbmanager,
service: service,
tenant: tenant,
replicationType: replicationType,
appService: as,
}, nil
}
//Build builds service, ingress and secret for each port
func (a *AppServiceBuild) Build() (*v1.K8sResources, error) {
ports, err := a.dbmanager.TenantServicesPortDao().GetPortsByServiceID(a.serviceID)
if err != nil {
return nil, fmt.Errorf("find service port from db error %s", err.Error())
}
crt, err := checkUpstreamPluginRelation(a.serviceID, a.dbmanager)
if err != nil {
return nil, fmt.Errorf("get service upstream plugin relation error, %s", err.Error())
}
pp := make(map[int32]int)
if crt {
pluginPorts, err := a.dbmanager.TenantServicesStreamPluginPortDao().GetPluginMappingPorts(
a.serviceID)
if err != nil {
return nil, fmt.Errorf("find upstream plugin mapping port error, %s", err.Error())
}
ports, pp, err = a.CreateUpstreamPluginMappingPort(ports, pluginPorts)
}
if err != nil {
return nil, fmt.Errorf("create upstream port error, %s", err.Error())
}
var services []*corev1.Service
var ingresses []*extensions.Ingress
var secrets []*corev1.Secret
if ports != nil && len(ports) > 0 {
for i := range ports {
port := ports[i]
if *port.IsInnerService {
services = append(services, a.createInnerService(port))
}
if *port.IsOuterService {
service := a.createOuterService(port)
services = append(services, service)
relContainerPort := pp[int32(port.ContainerPort)]
if relContainerPort == 0 {
relContainerPort = port.ContainerPort
}
ings, secrs, err := a.ApplyRules(port.ServiceID, relContainerPort, port.ContainerPort, service)
if err != nil {
logrus.Errorf("error applying rules: %s", err.Error())
return nil, err
}
ingresses = append(ingresses, ings...)
if secrs != nil {
secrets = append(secrets, secrs...)
}
}
}
}
// build stateful service
if a.replicationType == model.TypeStatefulSet {
services = append(services, a.createStatefulService(ports))
}
if crt {
services, _ = a.CreateUpstreamPluginMappingService(services, pp)
}
return &v1.K8sResources{
Services: services,
Secrets: secrets,
Ingresses: ingresses,
}, nil
}
// ApplyRules applies http rules and tcp rules
func (a AppServiceBuild) ApplyRules(serviceID string, containerPort, pluginContainerPort int,
service *corev1.Service) ([]*extensions.Ingress, []*corev1.Secret, error) {
var ingresses []*extensions.Ingress
var secrets []*corev1.Secret
httpRules, err := a.dbmanager.HTTPRuleDao().GetHTTPRuleByServiceIDAndContainerPort(serviceID, containerPort)
if err != nil {
logrus.Infof("Can't get HTTPRule corresponding to ServiceID(%s): %v", serviceID, err)
}
// create http ingresses
logrus.Debugf("find %d count http rule", len(httpRules))
if httpRules != nil && len(httpRules) > 0 {
for _, httpRule := range httpRules {
ing, sec, err := a.applyHTTPRule(httpRule, containerPort, pluginContainerPort, service)
if err != nil {
logrus.Errorf("Unexpected error occurred while applying http rule: %v", err)
// skip the failed rule
continue
}
logrus.Debugf("create ingress %s", ing.Name)
ingresses = append(ingresses, ing)
secrets = append(secrets, sec)
}
}
// create tcp ingresses
tcpRules, err := a.dbmanager.TCPRuleDao().GetTCPRuleByServiceIDAndContainerPort(serviceID, containerPort)
if err != nil {
logrus.Infof("Can't get TCPRule corresponding to ServiceID(%s): %v", serviceID, err)
}
if tcpRules != nil && len(tcpRules) > 0 {
for _, tcpRule := range tcpRules {
ing, err := a.applyTCPRule(tcpRule, service, a.tenant.UUID)
if err != nil {
logrus.Errorf("Unexpected error occurred while applying tcp rule: %v", err)
// skip the failed rule
continue
}
ingresses = append(ingresses, ing)
}
}
return ingresses, secrets, nil
}
// applyTCPRule applies stream rule into ingress
func (a *AppServiceBuild) applyHTTPRule(rule *model.HTTPRule, containerPort, pluginContainerPort int,
service *corev1.Service) (ing *extensions.Ingress, sec *corev1.Secret, err error) {
// deal with empty path and domain
path := strings.Replace(rule.Path, " ", "", -1)
if path == "" {
path = "/"
}
domain := strings.Replace(rule.Domain, " ", "", -1)
if domain == "" {
domain = createDefaultDomain(a.tenant.Name, a.service.ServiceAlias, containerPort)
}
// create ingress
ing = &extensions.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: rule.UUID,
Namespace: a.tenant.UUID,
Labels: a.appService.GetCommonLabels(),
},
Spec: extensions.IngressSpec{
Rules: []extensions.IngressRule{
{
Host: domain,
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: path,
Backend: extensions.IngressBackend{
ServiceName: service.Name,
ServicePort: intstr.FromInt(pluginContainerPort),
},
},
},
},
},
},
},
},
}
// parse annotations
annos := make(map[string]string)
// weight
if rule.Weight > 1 {
annos[parser.GetAnnotationWithPrefix("weight")] = fmt.Sprintf("%d", rule.Weight)
}
// header
if rule.Header != "" {
annos[parser.GetAnnotationWithPrefix("header")] = rule.Header
}
// cookie
if rule.Cookie != "" {
annos[parser.GetAnnotationWithPrefix("cookie")] = rule.Cookie
}
// certificate
if rule.CertificateID != "" {
cert, err := a.dbmanager.CertificateDao().GetCertificateByID(rule.CertificateID)
if err != nil {
return nil, nil, fmt.Errorf("Cant not get certificate by id(%s): %v", rule.CertificateID, err)
}
if cert == nil || strings.TrimSpace(cert.Certificate) == "" || strings.TrimSpace(cert.PrivateKey) == "" {
return nil, nil, fmt.Errorf("Rule id: %s; certificate not found", rule.UUID)
}
// create secret
sec = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: rule.UUID, // TODO: one cert, one secret
Namespace: a.tenant.UUID,
Labels: a.appService.GetCommonLabels(),
},
Data: map[string][]byte{
"tls.crt": []byte(cert.Certificate),
"tls.key": []byte(cert.PrivateKey),
},
Type: corev1.SecretTypeOpaque,
}
ing.Spec.TLS = []extensions.IngressTLS{
{
Hosts: []string{domain},
SecretName: sec.Name,
},
}
}
// rule extension
ruleExtensions, err := a.dbmanager.RuleExtensionDao().GetRuleExtensionByRuleID(rule.UUID)
if err != nil {
return nil, nil, err
}
for _, extension := range ruleExtensions {
switch extension.Key {
case string(model.HTTPToHTTPS):
if rule.CertificateID == "" {
logrus.Warningf("enable force-ssl-redirect, but with no certificate. rule id is: %s", rule.UUID)
break
}
annos[parser.GetAnnotationWithPrefix("force-ssl-redirect")] = "true"
case string(model.LBType):
if strings.HasPrefix(extension.Value, "upstream-hash-by") {
s := strings.Split(extension.Value, ":")
if len(s) < 2 {
logrus.Warningf("invalid extension value for upstream-hash-by: %s", extension.Value)
break
}
annos[parser.GetAnnotationWithPrefix("upstream-hash-by")] = s[1]
break
}
annos[parser.GetAnnotationWithPrefix("lb-type")] = extension.Value
default:
logrus.Warnf("Unexpected RuleExtension Key: %s", extension.Key)
}
}
configs, err := db.GetManager().GwRuleConfigDao().ListByRuleID(rule.UUID)
if err != nil {
return nil, nil, err
}
if configs != nil && len(configs) > 0 {
for _, cfg := range configs {
annos[parser.GetAnnotationWithPrefix(cfg.Key)] = cfg.Value
}
}
ing.SetAnnotations(annos)
return ing, sec, nil
}
// applyTCPRule applies stream rule into ingress
func (a *AppServiceBuild) applyTCPRule(rule *model.TCPRule, service *corev1.Service, namespace string) (ing *extensions.Ingress, err error) {
// create ingress
ing = &extensions.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: rule.UUID,
Namespace: namespace,
Labels: a.appService.GetCommonLabels(),
},
Spec: extensions.IngressSpec{
Backend: &extensions.IngressBackend{
ServiceName: service.Name,
ServicePort: intstr.FromInt(int(service.Spec.Ports[0].Port)),
},
},
}
annos := make(map[string]string)
annos[parser.GetAnnotationWithPrefix("l4-enable")] = "true"
annos[parser.GetAnnotationWithPrefix("l4-host")] = rule.IP
annos[parser.GetAnnotationWithPrefix("l4-port")] = fmt.Sprintf("%v", rule.Port)
ing.SetAnnotations(annos)
return ing, nil
}
//CreateUpstreamPluginMappingPort 检查是否存在upstream插件接管入口网络
func (a *AppServiceBuild) CreateUpstreamPluginMappingPort(
ports []*model.TenantServicesPort,
pluginPorts []*model.TenantServicesStreamPluginPort,
) (
[]*model.TenantServicesPort,
map[int32]int,
error) {
//start from 65301
pp := make(map[int32]int)
for i := range ports {
port := ports[i]
for _, pport := range pluginPorts {
if pport.ContainerPort == port.ContainerPort {
pp[int32(pport.PluginPort)] = port.ContainerPort
port.ContainerPort = pport.PluginPort
port.MappingPort = pport.PluginPort
}
}
}
return ports, pp, nil
}
//CreateUpstreamPluginMappingService 增加service plugin mapport 标签
func (a *AppServiceBuild) CreateUpstreamPluginMappingService(services []*corev1.Service,
pp map[int32]int) ([]*corev1.Service, error) {
for _, service := range services {
logrus.Debugf("map is %v, port is %v, origin_port is %d",
pp,
service.Spec.Ports[0].Port,
pp[service.Spec.Ports[0].Port])
service.Labels["origin_port"] = fmt.Sprintf("%d", pp[service.Spec.Ports[0].Port])
}
return services, nil
}
//BuildOnPort 指定端口创建Service
func (a *AppServiceBuild) BuildOnPort(p int, isOut bool) (*corev1.Service, error) {
port, err := a.dbmanager.TenantServicesPortDao().GetPort(a.serviceID, p)
if err != nil {
return nil, fmt.Errorf("find service port from db error %s", err.Error())
}
if port != nil {
if !isOut && *port.IsInnerService {
return a.createInnerService(port), nil
}
if isOut && *port.IsOuterService {
return a.createOuterService(port), nil
}
}
return nil, fmt.Errorf("tenant service port %d is not exist", p)
}
//createServiceAnnotations create service annotation
func (a *AppServiceBuild) createServiceAnnotations() map[string]string {
var annotations = make(map[string]string)
if a.service.Replicas <= 1 {
annotations["rainbond.com/tolerate-unready-endpoints"] = "true"
}
return annotations
}
func (a *AppServiceBuild) createInnerService(port *model.TenantServicesPort) *corev1.Service {
var service corev1.Service
service.Name = fmt.Sprintf("service-%d-%d", port.ID, port.ContainerPort)
service.Namespace = a.service.TenantID
service.Labels = a.appService.GetCommonLabels(map[string]string{
"service_type": "inner",
"name": a.service.ServiceAlias + "Service",
"port_protocol": port.Protocol,
"version": a.service.DeployVersion,
})
if a.service.Replicas <= 1 {
service.Labels["rainbond.com/tolerate-unready-endpoints"] = "true"
}
service.Annotations = a.createServiceAnnotations()
var servicePort corev1.ServicePort
if port.Protocol == "udp" {
servicePort.Protocol = "UDP"
} else {
servicePort.Protocol = "TCP"
}
servicePort.TargetPort = intstr.FromInt(port.ContainerPort)
servicePort.Port = int32(port.MappingPort)
if servicePort.Port == 0 {
servicePort.Port = int32(port.ContainerPort)
}
spec := corev1.ServiceSpec{
Ports: []corev1.ServicePort{servicePort},
}
if a.appService.ServiceKind != model.ServiceKindThirdParty {
spec.Selector = map[string]string{"name": a.service.ServiceAlias}
}
service.Spec = spec
return &service
}
func (a *AppServiceBuild) createOuterService(port *model.TenantServicesPort) *corev1.Service {
var service corev1.Service
service.Name = fmt.Sprintf("service-%d-%dout", port.ID, port.ContainerPort)
service.Namespace = a.service.TenantID
service.Labels = a.appService.GetCommonLabels(map[string]string{
"service_type": "outer",
"name": a.service.ServiceAlias + "ServiceOUT",
"tenant_name": a.tenant.Name,
"protocol": port.Protocol,
"port_protocol": port.Protocol,
"event_id": a.eventID,
"version": a.service.DeployVersion,
})
if a.service.Replicas <= 1 {
service.Labels["rainbond.com/tolerate-unready-endpoints"] = "true"
}
var servicePort corev1.ServicePort
servicePort.Protocol = conversionPortProtocol(port.Protocol)
servicePort.TargetPort = intstr.FromInt(port.ContainerPort)
servicePort.Port = int32(port.ContainerPort)
var portType corev1.ServiceType
if os.Getenv("CUR_NET") == "midonet" {
portType = corev1.ServiceTypeNodePort
} else {
portType = corev1.ServiceTypeClusterIP
}
spec := corev1.ServiceSpec{
Ports: []corev1.ServicePort{servicePort},
Type: portType,
}
if a.appService.ServiceKind != model.ServiceKindThirdParty {
spec.Selector = map[string]string{"name": a.service.ServiceAlias}
}
service.Spec = spec
return &service
}
func (a *AppServiceBuild) createStatefulService(ports []*model.TenantServicesPort) *corev1.Service {
var service corev1.Service
service.Name = a.service.ServiceName
service.Namespace = a.service.TenantID
if service.Name == "" {
service.Name = a.service.ServiceAlias
}
service.Labels = a.appService.GetCommonLabels(map[string]string{
"service_type": "stateful",
"name": a.service.ServiceAlias + "ServiceStateful",
})
var serviceports []corev1.ServicePort
for _, p := range ports {
var servicePort corev1.ServicePort
servicePort.Protocol = "TCP"
servicePort.TargetPort = intstr.FromInt(p.ContainerPort)
servicePort.Port = int32(p.MappingPort)
servicePort.Name = fmt.Sprintf("%d-port", p.ID)
if servicePort.Port == 0 {
servicePort.Port = int32(p.ContainerPort)
}
serviceports = append(serviceports, servicePort)
}
spec := corev1.ServiceSpec{
Ports: serviceports,
Selector: map[string]string{"name": a.service.ServiceAlias},
ClusterIP: "None",
PublishNotReadyAddresses: true,
}
service.Spec = spec
service.Annotations = map[string]string{"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true"}
return &service
}