[ADD] add gateway Dockerfile

Makefile

@@ -16,13 +16,16 @@ build_items="api chaos entrance monitor mq node webcli worker eventlog"
 ifeq ($(origin WHAT), undefined)
   WHAT = all
 endif
+ifeq ($(origin PUSH), undefined)
+  PUSH = false
+endif
 .PHONY: build
 build:
 	@echo "🐳build ${WHAT}" 
 	@./localbuild.sh $(WHAT)
 image:
 	@echo "🐳build image ${WHAT}" 	
-	@bash ./release.sh ${WHAT}
+	@bash ./release.sh ${WHAT} ${PUSH}
 
 run:build
 ifeq ($(WHAT),api)

cmd/gateway/option/option.go

@@ -41,6 +41,17 @@ type Config struct {
 	K8SConfPath string
 	Namespace   string
 	ListenPorts ListenPorts
+	//This number should be, at maximum, the number of CPU cores on your system.
+	WorkerProcesses    int
+	WorkerRlimitNofile int
+	ErrorLog           string
+	WorkerConnections  int
+	//essential for linux, optmized to serve many clients with each thread
+	EnableEpool       bool
+	EnableMultiAccept bool
+	KeepaliveTimeout  int
+	KeepaliveRequests int
+	NginxUser         string
 }
 
 // ListenPorts describe the ports required to run the gateway controller
@@ -58,6 +69,15 @@ func (g *GWServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&g.K8SConfPath, "kube-conf", "/opt/rainbond/etc/kubernetes/kubecfg/admin.kubeconfig", "absolute path to the kubeconfig file")
 	fs.StringVar(&g.Namespace, "namespace", "gateway", "namespace")
 	fs.IntVar(&g.ListenPorts.AuxiliaryPort, "auxiliary-port", 10253, "port of auxiliary server")
+	fs.IntVar(&g.WorkerProcesses, "worker-processes", 0, "Default get current compute cpu core number.This number should be, at maximum, the number of CPU cores on your system.")
+	fs.IntVar(&g.WorkerConnections, "worker-connections", 4000, "Determines how many clients will be served by each worker process.")
+	fs.IntVar(&g.WorkerRlimitNofile, "worker-rlimit-nofile", 200000, "Number of file descriptors used for Nginx. This is set in the OS with 'ulimit -n 200000'")
+	fs.BoolVar(&g.EnableEpool, "enable-epool", true, "essential for linux, optmized to serve many clients with each thread")
+	fs.BoolVar(&g.EnableMultiAccept, "enable-multi-accept", true, "Accept as many connections as possible, after nginx gets notification about a new connection.")
+	fs.StringVar(&g.ErrorLog, "error-log", "/dev/stderr crit", "only log critical errors")
+	fs.StringVar(&g.NginxUser, "nginx-user", "root", "nginx user name")
+	fs.IntVar(&g.KeepaliveRequests, "keepalive-requests", 100000, "Number of requests a client can make over the keep-alive connection. This is set high for testing.")
+	fs.IntVar(&g.KeepaliveTimeout, "keepalive-timeout", 30, "Timeout for keep-alive connections. Server will close connections after this time.")
 }
 
 // SetLog sets log

gateway/controller/controller.go

@@ -2,6 +2,9 @@ package controller
 
 import (
 	"fmt"
+	"sync"
+	"time"
+
 	"github.com/Sirupsen/logrus"
 	"github.com/eapache/channels"
 	"github.com/golang/glog"
@@ -11,8 +14,6 @@ import (
 	"github.com/goodrain/rainbond/gateway/v1"
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/ingress-nginx/task"
-	"sync"
-	"time"
 )
 
 const (
@@ -170,6 +171,7 @@ func (gwc *GWController) getDelUpdPools(updPools []*v1.Pool) ([]*v1.Pool, []*v1.
 	return delPools, updPools
 }
 
+//NewGWController new Gateway controller
 func NewGWController(config *option.Config, errCh chan error) *GWController {
 	logrus.Debug("NewGWController...")
 	gwc := &GWController{
@@ -179,10 +181,7 @@ func NewGWController(config *option.Config, errCh chan error) *GWController {
 		stopCh:   make(chan struct{}),
 	}
 
-	gws := &openresty.OpenrestyService{
-		AuxiliaryPort:  config.ListenPorts.AuxiliaryPort,
-		IsShuttingDown: &gwc.isShuttingDown,
-	}
+	gws := openresty.CreateOpenrestyService(config, &gwc.isShuttingDown)
 	gwc.GWS = gws
 
 	clientSet, err := NewClientSet(config.K8SConfPath)

gateway/controller/openresty/model/http.go

@@ -1,9 +1,12 @@
 package model
 
+import "github.com/goodrain/rainbond/cmd/gateway/option"
+
 type Http struct {
 	DefaultType          string
 	SendFile             bool
 	KeepaliveTimeout     Time
+	keepaliveRequests    int
 	ClientMaxBodySize    Size
 	ClientBodyBufferSize Size
 	ProxyConnectTimeout  Time
@@ -27,14 +30,15 @@ type AccessLog struct {
 }
 
 // NewHttp creates a new model.Http
-func NewHttp() *Http {
+func NewHttp(conf option.Config) *Http {
 	return &Http{
 		DefaultType: "text/html",
 		SendFile:    true,
 		KeepaliveTimeout: Time{
-			Num:  70,
+			Num:  conf.KeepaliveTimeout,
 			Unit: "s",
 		},
+		keepaliveRequests: conf.KeepaliveRequests,
 		ClientMaxBodySize: Size{
 			Num:  10,
 			Unit: "m",
@@ -63,8 +67,6 @@ func NewHttp() *Http {
 			Num:  32,
 			Unit: "k",
 		},
-		Includes: []string{
-			"/export/servers/nginx/conf/servers-http.conf",
-		},
+		Includes: []string{},
 	}
 }

gateway/controller/openresty/model/nginx.go

@@ -1,10 +1,22 @@
 package model
 
+import (
+	"os/user"
+	"path"
+	"runtime"
+
+	"github.com/goodrain/rainbond/cmd/gateway/option"
+)
+
+//Nginx nginx config model
 type Nginx struct {
-	WorkerProcesses int
-	EventLog        EventLog
-	Events          Events
-	Includes        []string
+	WorkerProcesses    int
+	WorkerRlimitNofile int
+	ErrorLog           string
+	User               string
+	EventLog           EventLog
+	Events             Events
+	Includes           []string
 }
 
 type EventLog struct {
@@ -12,15 +24,36 @@ type EventLog struct {
 	Level string
 }
 
+//Events nginx events config model
 type Events struct {
 	WorkerConnections int
+	EnableEpoll       bool
+	EnableMultiAccept bool
 }
 
-func NewNginx() *Nginx {
+//NewNginx new nginx config
+func NewNginx(conf option.Config, customPath string) *Nginx {
+	if conf.NginxUser != "" {
+		if u, err := user.Current(); err == nil {
+			if conf.NginxUser == u.Username {
+				//if set user name like run user,do not set
+				conf.NginxUser = ""
+			}
+		}
+	}
+	if conf.WorkerProcesses == 0 {
+		conf.WorkerProcesses = runtime.NumCPU()
+	}
 	return &Nginx{
-		WorkerProcesses: 2, // TODO
-		Includes: []string{
-			"/export/servers/nginx/conf/http.conf",
+		WorkerProcesses:    conf.WorkerProcesses,
+		WorkerRlimitNofile: conf.WorkerRlimitNofile,
+		Includes:           []string{path.Join(customPath, "/*.conf")},
+		User:               conf.NginxUser,
+		ErrorLog:           conf.ErrorLog,
+		Events: Events{
+			WorkerConnections: conf.WorkerConnections,
+			EnableEpoll:       conf.EnableEpool,
+			EnableMultiAccept: conf.EnableMultiAccept,
 		},
 	}
 }

gateway/controller/openresty/nginx_cmd.go

@@ -3,22 +3,30 @@ package openresty
 import (
 	"os"
 	"os/exec"
+	"path"
+
+	"github.com/goodrain/rainbond/gateway/controller/openresty/template"
 )
 
-const (
-	defBinary = "/usr/local/opt/openresty/nginx/sbin/nginx"
-	cfgPath   = "/export/Servers/nginx/conf/nginx.conf"
+var (
+	nginxBinary      = "nginx"
+	defaultNginxConf = "/run/nginx/nginx.conf"
 )
 
-func nginxExecCommand(args ...string) *exec.Cmd {
+func init() {
+	nginxBinary = path.Join(os.Getenv("OPENRESTY_HOME"), "/nginx/sbin/nginx")
 	ngx := os.Getenv("NGINX_BINARY")
-	if ngx == "" {
-		ngx = defBinary
+	if ngx != "" {
+		nginxBinary = ngx
+	}
+	customConfig := os.Getenv("NGINX_CUSTOM_CONFIG")
+	if customConfig != "" {
+		template.CustomConfigPath = customConfig
 	}
-
-	var cmdArgs []string
-	cmdArgs = append(cmdArgs,"-c", cfgPath)
-	cmdArgs = append(cmdArgs, args...)
-
-	return exec.Command(ngx, cmdArgs...)
+}
+func nginxExecCommand(args ...string) *exec.Cmd {
+	var cmdArgs []string
+	cmdArgs = append(cmdArgs, "-c", defaultNginxConf)
+	cmdArgs = append(cmdArgs, args...)
+	return exec.Command(nginxBinary, cmdArgs...)
 }

gateway/controller/openresty/service.go

@@ -4,27 +4,41 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/Sirupsen/logrus"
-	"github.com/goodrain/rainbond/gateway/controller/openresty/model"
-	"github.com/goodrain/rainbond/gateway/controller/openresty/template"
-	"github.com/goodrain/rainbond/gateway/v1"
 	"io/ioutil"
-	"k8s.io/ingress-nginx/ingress/controller/process"
 	"net/http"
 	"os"
+	"path"
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/cmd/gateway/option"
+	"github.com/goodrain/rainbond/gateway/controller/openresty/model"
+	"github.com/goodrain/rainbond/gateway/controller/openresty/template"
+	"github.com/goodrain/rainbond/gateway/v1"
+	"k8s.io/ingress-nginx/ingress/controller/process"
 )
 
-type OpenrestyService struct{
-	AuxiliaryPort int
+type OpenrestyService struct {
+	AuxiliaryPort  int
 	IsShuttingDown *bool
 
 	// stopLock is used to enforce that only a single call to Stop send at
 	// a given time. We allow stopping through an HTTP endpoint and
 	// allowing concurrent stoppers leads to stack traces.
 	stopLock *sync.Mutex
+	config   *option.Config
+}
+
+//CreateOpenrestyService create openresty service
+func CreateOpenrestyService(config *option.Config, isShuttingDown *bool) *OpenrestyService {
+	gws := &OpenrestyService{
+		AuxiliaryPort:  config.ListenPorts.AuxiliaryPort,
+		IsShuttingDown: isShuttingDown,
+		config:         config,
+	}
+	return gws
 }
 
 type Upstream struct {
@@ -37,7 +51,12 @@ type Server struct {
 	Port int32
 }
 
+//Start start
 func (osvc *OpenrestyService) Start() error {
+	nginx := model.NewNginx(*osvc.config, template.CustomConfigPath)
+	if err := template.NewNginxTemplate(nginx, defaultNginxConf); err != nil {
+		return err
+	}
 	o, err := nginxExecCommand().CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("%v\n%v", err, string(o))
@@ -82,7 +101,6 @@ func (osvc *OpenrestyService) PersistConfig(conf *v1.Config) error {
 	}
 
 	l7srv, l4srv := getNgxServer(conf)
-	var ngxIncludes []string
 	// http
 	if len(l7srv) > 0 {
 		serverFilename := "servers-http.conf"
@@ -91,17 +109,16 @@ func (osvc *OpenrestyService) PersistConfig(conf *v1.Config) error {
 			return err
 		}
 
-		httpData := model.NewHttp()
+		httpData := model.NewHttp(*osvc.config)
 		httpData.Includes = []string{
-			serverFilename,
-			upsHttp,
+			path.Join(template.CustomConfigPath, serverFilename),
+			path.Join(template.CustomConfigPath, upsHttp),
 		}
 		httpFilename := "http.conf"
 		if err := template.NewHttpTemplate(httpData, httpFilename); err != nil {
 			logrus.Fatalf("Fail to new nginx http template: %v", err)
 			return nil
 		}
-		ngxIncludes = append(ngxIncludes, httpFilename)
 	}
 
 	// stream
@@ -113,21 +130,14 @@ func (osvc *OpenrestyService) PersistConfig(conf *v1.Config) error {
 		}
 		streamData := model.NewStream()
 		streamData.Includes = []string{
-			serverFilename,
-			upsTcp,
+			path.Join(template.CustomConfigPath, serverFilename),
+			path.Join(template.CustomConfigPath, upsTcp),
 		}
 		streamFilename := "stream.conf"
 		if err := template.NewStreamTemplate(streamData, streamFilename); err != nil {
 			logrus.Fatalf("Fail to new nginx stream template: %v", err)
 			return nil
 		}
-		ngxIncludes = append(ngxIncludes, streamFilename)
-	}
-
-	nginx := model.NewNginx()
-	nginx.Includes = ngxIncludes
-	if err := template.NewNginxTemplate(nginx); err != nil {
-		return err
 	}
 
 	// check nginx configuration
@@ -185,8 +195,8 @@ func getNgxServer(conf *v1.Config) (l7srv []*model.Server, l4srv []*model.Server
 		}
 		for _, loc := range vs.Locations {
 			location := &model.Location{
-				Path:      loc.Path,
-				NameCondition:loc.NameCondition,
+				Path:          loc.Path,
+				NameCondition: loc.NameCondition,
 			}
 			server.Locations = append(server.Locations, location)
 		}

gateway/controller/openresty/template/template.go

@@ -18,22 +18,28 @@ package template
 
 import (
 	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+	text_template "text/template"
+
 	"github.com/golang/glog"
 	"github.com/goodrain/rainbond/gateway/controller/openresty/model"
 	"github.com/pkg/errors"
-	"io/ioutil"
-	"os"
-	text_template "text/template"
 )
 
-const (
+var (
 	defBufferSize = 65535
-	ConfPath      = "/export/servers/nginx/conf"
-	tmplPath      = "/export/servers/nginx/tmpl"
+	//CustomConfigPath custom config file path
+	CustomConfigPath = "/run/nginx/conf"
+	//tmplPath Tmpl config file path
+	tmplPath = "/run/nginxtmp"
 )
 
 func init() {
-
+	if os.Getenv("NGINX_CONFIG_TMPL") != "" {
+		tmplPath = os.Getenv("NGINX_CONFIG_TMPL")
+	}
 }
 
 // Template ...
@@ -62,40 +68,40 @@ func NewTemplate(fileName string) (*Template, error) {
 }
 
 // NewNginxTemplate creates a nginx configuration file(nginx.conf)
-func NewNginxTemplate(data *model.Nginx) error {
-	if e := Persist(tmplPath+"/nginx.tmpl", data, ConfPath, "nginx.conf"); e != nil {
+func NewNginxTemplate(data *model.Nginx, defaultNginxConf string) error {
+	if e := Persist(tmplPath+"/nginx.tmpl", data, path.Dir(defaultNginxConf), path.Base(defaultNginxConf)); e != nil {
 		return e
 	}
 	return nil
 }
 
-// NewNginxTemplate creates a configuration file for the nginx http module
+// NewHttpTemplate creates a configuration file for the nginx http module
 func NewHttpTemplate(data *model.Http, filename string) error {
-	if e := Persist(tmplPath+"/http.tmpl", data, ConfPath, filename); e != nil {
+	if e := Persist(tmplPath+"/http.tmpl", data, CustomConfigPath, filename); e != nil {
 		return e
 	}
 	return nil
 }
 
-// NewNginxTemplate creates a configuration file for the nginx stream module
+// NewStreamTemplate creates a configuration file for the nginx stream module
 func NewStreamTemplate(data *model.Stream, filename string) error {
-	if e := Persist(tmplPath+"/stream.tmpl", data, ConfPath, filename); e != nil {
+	if e := Persist(tmplPath+"/stream.tmpl", data, CustomConfigPath, filename); e != nil {
 		return e
 	}
 	return nil
 }
 
-// NewNginxTemplate creates a configuration file for the nginx server module
+// NewServerTemplate creates a configuration file for the nginx server module
 func NewServerTemplate(data []*model.Server, filename string) error {
-	if e := Persist(tmplPath+"/servers.tmpl", data, ConfPath, filename); e != nil {
+	if e := Persist(tmplPath+"/servers.tmpl", data, CustomConfigPath, filename); e != nil {
 		return e
 	}
 	return nil
 }
 
-// NewNginxTemplate creates a configuration file for the nginx upstream module
+// NewUpstreamTemplate creates a configuration file for the nginx upstream module
 func NewUpstreamTemplate(data []model.Upstream, tmpl, filename string) error {
-	if e := Persist(tmplPath+"/"+tmpl, data, ConfPath, filename); e != nil {
+	if e := Persist(tmplPath+"/"+tmpl, data, CustomConfigPath, filename); e != nil {
 		return e
 	}
 	return nil
@@ -112,7 +118,6 @@ func Persist(tmplFilename string, data interface{}, path string, filename string
 	if err != nil {
 		return err
 	}
-
 	if e := os.MkdirAll(path, 0777); e != nil {
 		return e
 	}

gateway/rootfs/export/servers/nginx/tmpl/nginx.tmpl

@@ -1,11 +0,0 @@
-worker_processes  {{.WorkerProcesses}};
-
-error_log  /var/log/nginx/error.log;
-
-events {
-    worker_connections  2048;
-}
-
-{{ range $include := .Includes }}
-include {{$include}};
-{{ end }}

hack/contrib/docker/gateway/Dockerfile

@@ -3,7 +3,11 @@ FROM goodrainapps/alpine:3.4
 ADD rainbond-gateway /run/rainbond-gateway
 ADD entrypoint.sh /run/entrypoint.sh
 ADD openresty-1.13.6.1.tar.gz /
+ADD nginxtmp/tmpl /run/nginxtmp
+ADD nginxtmp/conf/mime.types /run/nginx/conf/mime.types
 
+ENV NGINX_CONFIG_TMPL=/run/nginxtmp
+ENV NGINX_CUSTOM_CONFIG=/run/nginx/conf
 ENV RELEASE_DESC=__RELEASE_DESC__
 ENV OPENRESTY_HOME=/usr/local/openresty
 ENV PATH="${PATH}:${OPENRESTY_HOME}/nginx/sbin"

hack/contrib/docker/gateway/Dockerfile.release

@@ -1,13 +0,0 @@
-FROM goodrainapps/alpine:3.4
-
-ADD rainbond-gateway /run/rainbond-gateway
-ADD entrypoint.sh /run/entrypoint.sh
-ADD openresty-1.13.6.1.tar.gz /
-
-ENV RELEASE_DESC=5.0-8eea0849-2018-11-21-13
-ENV OPENRESTY_HOME=/usr/local/openresty
-ENV PATH="${PATH}:${OPENRESTY_HOME}/nginx/sbin"
-
-EXPOSE 8080
-
-ENTRYPOINT ["/run/entrypoint.sh"]

hack/contrib/docker/gateway/nginxtmp/tmpl/nginx.tmpl

@@ -0,0 +1,15 @@
+
+{{ if .User }} user {{.User}};{{ end }}
+worker_processes  {{.WorkerProcesses}};
+error_log  {{.ErrorLog}};
+worker_rlimit_nofile {{.WorkerRlimitNofile}};
+
+events {
+    {{ if .Events.EnableEpoll }}use epoll;{{ end }}
+    {{ if .Events.EnableMultiAccept }}multi_accept on;{{ end }}
+    worker_connections  {{.Events.WorkerConnections}};
+}
+
+{{ range $include := .Includes }}
+include {{$include}};
+{{ end }}

node/nodem/healthy/health_manager.go

@@ -104,6 +104,9 @@ func (p *probeManager) AddServices(inner []*service.Service) error {
 func (p *probeManager) Start(hostNode *client.HostNode) error {
 	go p.HandleStatus()
 	for _, v := range p.services {
+		if v.ServiceHealth == nil {
+			continue
+		}
 		if v.ServiceHealth.Model == "http" {
 			h := &probe.HttpProbe{
 				Name:         v.ServiceHealth.Name,

release.sh

@@ -70,7 +70,9 @@ build::image() {
 		echo "---> build image:$1"
 		sed "s/__RELEASE_DESC__/${release_desc}/" Dockerfile > Dockerfile.release
 		docker build -t ${BASE_NAME}/rbd-$1:${VERSION} -f Dockerfile.release .
-		docker push ${BASE_NAME}/rbd-$1:${VERSION}
+		if [ "$2" = "true" ];then
+			docker push ${BASE_NAME}/rbd-$1:${VERSION}
+		fi	
 		rm -f ./Dockerfile.release
 		rm -f ./${BASE_NAME}-$1
 	popd
@@ -80,7 +82,7 @@ build::all(){
 	local build_items=(api chaos entrance monitor mq webcli worker eventlog)
 	for item in ${build_items[@]}
 	do
-		build::image $item
+		build::image $item $1
 	done
 	build::node
 }
@@ -91,9 +93,9 @@ case $1 in
 	;;
 	*)
 		if [ "$1" = "all" ];then
-			build::all
+			build::all $2
 		else
-			build::image $1
+			build::image $1 $2
 		fi
 	;;
 esac