diff --git a/gateway/annotations/proxy/proxy.go b/gateway/annotations/proxy/proxy.go index dc00f6427..72077bda8 100644 --- a/gateway/annotations/proxy/proxy.go +++ b/gateway/annotations/proxy/proxy.go @@ -32,21 +32,22 @@ import ( // Config returns the proxy timeout to use in the upstream server/s type Config struct { - BodySize int `json:"bodySize"` - ConnectTimeout int `json:"connectTimeout"` - SendTimeout int `json:"sendTimeout"` - ReadTimeout int `json:"readTimeout"` - BuffersNumber int `json:"buffersNumber"` - BufferSize string `json:"bufferSize"` - CookieDomain string `json:"cookieDomain"` - CookiePath string `json:"cookiePath"` - NextUpstream string `json:"nextUpstream"` - NextUpstreamTries int `json:"nextUpstreamTries"` - ProxyRedirectFrom string `json:"proxyRedirectFrom"` - ProxyRedirectTo string `json:"proxyRedirectTo"` - RequestBuffering string `json:"requestBuffering"` - ProxyBuffering string `json:"proxyBuffering"` - SetHeaders map[string]string `json:"setHeaders"` + BodySize int `json:"bodySize"` + ConnectTimeout int `json:"connectTimeout"` + SendTimeout int `json:"sendTimeout"` + ReadTimeout int `json:"readTimeout"` + BuffersNumber int `json:"buffersNumber"` + BufferSize string `json:"bufferSize"` + CookieDomain string `json:"cookieDomain"` + CookiePath string `json:"cookiePath"` + NextUpstream string `json:"nextUpstream"` + NextUpstreamTimeout int `json:"nextUpstreamTimeout"` + NextUpstreamTries int `json:"nextUpstreamTries"` + ProxyRedirectFrom string `json:"proxyRedirectFrom"` + ProxyRedirectTo string `json:"proxyRedirectTo"` + RequestBuffering string `json:"requestBuffering"` + ProxyBuffering string `json:"proxyBuffering"` + SetHeaders map[string]string `json:"setHeaders"` } //Validation validation nginx parameters @@ -85,21 +86,22 @@ func (s *Config) Validation() error { func NewProxyConfig() Config { defBackend := config.NewDefault() return Config{ - BodySize: defBackend.ProxyBodySize, - ConnectTimeout: defBackend.ProxyConnectTimeout, - SendTimeout: defBackend.ProxySendTimeout, - ReadTimeout: defBackend.ProxyReadTimeout, - BuffersNumber: defBackend.ProxyBuffersNumber, - BufferSize: defBackend.ProxyBufferSize, - CookieDomain: defBackend.ProxyCookieDomain, - CookiePath: defBackend.ProxyCookiePath, - NextUpstream: defBackend.ProxyNextUpstream, - NextUpstreamTries: defBackend.ProxyNextUpstreamTries, - RequestBuffering: defBackend.ProxyRequestBuffering, - ProxyRedirectFrom: defBackend.ProxyRedirectFrom, - ProxyRedirectTo: defBackend.ProxyRedirectTo, - ProxyBuffering: defBackend.ProxyBuffering, - SetHeaders: defBackend.ProxySetHeaders, + BodySize: defBackend.ProxyBodySize, + ConnectTimeout: defBackend.ProxyConnectTimeout, + SendTimeout: defBackend.ProxySendTimeout, + ReadTimeout: defBackend.ProxyReadTimeout, + BuffersNumber: defBackend.ProxyBuffersNumber, + BufferSize: defBackend.ProxyBufferSize, + CookieDomain: defBackend.ProxyCookieDomain, + CookiePath: defBackend.ProxyCookiePath, + NextUpstream: defBackend.ProxyNextUpstream, + NextUpstreamTries: defBackend.ProxyNextUpstreamTries, + NextUpstreamTimeout: defBackend.ProxyNextUpstreamTimeout, + RequestBuffering: defBackend.ProxyRequestBuffering, + ProxyRedirectFrom: defBackend.ProxyRedirectFrom, + ProxyRedirectTo: defBackend.ProxyRedirectTo, + ProxyBuffering: defBackend.ProxyBuffering, + SetHeaders: defBackend.ProxySetHeaders, } } @@ -233,6 +235,11 @@ func (a proxy) Parse(ing *extensions.Ingress) (interface{}, error) { config.NextUpstreamTries = defBackend.ProxyNextUpstreamTries } + config.NextUpstreamTimeout, err = parser.GetIntAnnotation("proxy-next-upstream-timeout", ing) + if err != nil { + config.NextUpstreamTimeout = defBackend.ProxyNextUpstreamTimeout + } + config.RequestBuffering, err = parser.GetStringAnnotation("proxy-request-buffering", ing) if err != nil { config.RequestBuffering = defBackend.ProxyRequestBuffering diff --git a/gateway/annotations/proxy/proxy_test.go b/gateway/annotations/proxy/proxy_test.go index 23bdf2a83..06cb36ad7 100644 --- a/gateway/annotations/proxy/proxy_test.go +++ b/gateway/annotations/proxy/proxy_test.go @@ -70,16 +70,17 @@ type mockBackend struct { func (m mockBackend) GetDefaultBackend() defaults.Backend { return defaults.Backend{ - ProxyConnectTimeout: 10, - ProxySendTimeout: 15, - ProxyReadTimeout: 20, - ProxyBuffersNumber: 4, - ProxyBufferSize: "10k", - ProxyBodySize: 3, - ProxyNextUpstream: "error", - ProxyNextUpstreamTries: 3, - ProxyRequestBuffering: "on", - ProxyBuffering: "off", + ProxyConnectTimeout: 10, + ProxySendTimeout: 15, + ProxyReadTimeout: 20, + ProxyBuffersNumber: 4, + ProxyBufferSize: "10k", + ProxyBodySize: 3, + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTries: 3, + ProxyNextUpstreamTimeout: 0, + ProxyRequestBuffering: "on", + ProxyBuffering: "off", } } @@ -125,9 +126,6 @@ func TestProxy(t *testing.T) { if p.BodySize != 2 { t.Errorf("expected 2 as body-size but returned %v", p.BodySize) } - if p.NextUpstream != "off" { - t.Errorf("expected off as next-upstream but returned %v", p.NextUpstream) - } if p.NextUpstreamTries != 3 { t.Errorf("expected 3 as next-upstream-tries but returned %v", p.NextUpstreamTries) } @@ -171,9 +169,6 @@ func TestProxyWithNoAnnotation(t *testing.T) { if p.BodySize != 3 { t.Errorf("expected 3k as body-size but returned %v", p.BodySize) } - if p.NextUpstream != "error" { - t.Errorf("expected error as next-upstream but returned %v", p.NextUpstream) - } if p.NextUpstreamTries != 3 { t.Errorf("expected 3 as next-upstream-tries but returned %v", p.NextUpstreamTries) } diff --git a/gateway/controller/config/config.go b/gateway/controller/config/config.go index 89329436e..2e5f57518 100644 --- a/gateway/controller/config/config.go +++ b/gateway/controller/config/config.go @@ -80,26 +80,27 @@ type Configuration struct { func NewDefault() Configuration { cfg := Configuration{ Backend: defaults.Backend{ - ProxyBodySize: bodySize, - ProxyConnectTimeout: 60, - ProxyReadTimeout: 60, - ProxySendTimeout: 60, - ProxyBuffersNumber: 4, - ProxyBufferSize: "4k", - ProxyCookieDomain: "off", - ProxyCookiePath: "off", - ProxyNextUpstream: "error timeout", - ProxyNextUpstreamTries: 3, - ProxyRequestBuffering: "on", - ProxyRedirectFrom: "off", - ProxyRedirectTo: "off", - SSLRedirect: true, - CustomHTTPErrors: []int{}, - WhitelistSourceRange: []string{}, - SkipAccessLogURLs: []string{}, - LimitRate: 0, - LimitRateAfter: 0, - ProxyBuffering: "off", + ProxyBodySize: bodySize, + ProxyConnectTimeout: 5, + ProxyReadTimeout: 60, + ProxySendTimeout: 60, + ProxyBuffersNumber: 4, + ProxyBufferSize: "4k", + ProxyCookieDomain: "off", + ProxyCookiePath: "off", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTries: 3, + ProxyNextUpstreamTimeout: 0, + ProxyRequestBuffering: "on", + ProxyRedirectFrom: "off", + ProxyRedirectTo: "off", + SSLRedirect: true, + CustomHTTPErrors: []int{}, + WhitelistSourceRange: []string{}, + SkipAccessLogURLs: []string{}, + LimitRate: 0, + LimitRateAfter: 0, + ProxyBuffering: "off", //defaut set header ProxySetHeaders: map[string]string{ "Host": "$best_http_host", diff --git a/gateway/controller/openresty/model/server.go b/gateway/controller/openresty/model/server.go index a013366a3..02bcfd5f2 100644 --- a/gateway/controller/openresty/model/server.go +++ b/gateway/controller/openresty/model/server.go @@ -41,7 +41,10 @@ type Server struct { // Default: 1 ProxyStreamResponses int - ProxyStreamTimeout string + ProxyStreamTimeout string + ProxyStreamNextUpstream bool `json:"proxyStreamNextUpstream"` + ProxyStreamNextUpstreamTimeout string `json:"proxyStreamNextUpstreamTimeout"` + ProxyStreamNextUpstreamTries int `json:"proxyStreamNextUpstreamTries"` //proxy protocol for tcp real ip ProxyProtocol ProxyProtocol } diff --git a/gateway/controller/openresty/service.go b/gateway/controller/openresty/service.go index ce562b1db..f59fe6298 100644 --- a/gateway/controller/openresty/service.go +++ b/gateway/controller/openresty/service.go @@ -211,6 +211,9 @@ func (o *OrService) getNgxServer(conf *v1.Config) (l7srv []*model.Server, l4srv "tenant_id": vs.Namespace, "service_id": vs.ServiceID, }, + ProxyStreamNextUpstream: true, + ProxyStreamNextUpstreamTimeout: "600s", + ProxyStreamNextUpstreamTries: 3, } if vs.SSLCert != nil { server.SSLProtocols = vs.SSlProtocols @@ -223,14 +226,14 @@ func (o *OrService) getNgxServer(conf *v1.Config) (l7srv []*model.Server, l4srv location := &model.Location{ DisableAccessLog: o.ocfg.AccessLogPath == "", // TODO: Distinguish between server output logs - AccessLogPath: o.ocfg.AccessLogPath, - EnableMetrics: true, - Path: loc.Path, - NameCondition: loc.NameCondition, - Proxy: loc.Proxy, - Rewrite: loc.Rewrite, - PathRewrite: false, - DisableProxyPass: loc.DisableProxyPass, + AccessLogPath: o.ocfg.AccessLogPath, + EnableMetrics: true, + Path: loc.Path, + NameCondition: loc.NameCondition, + Proxy: loc.Proxy, + Rewrite: loc.Rewrite, + PathRewrite: false, + DisableProxyPass: loc.DisableProxyPass, } server.Locations = append(server.Locations, location) } @@ -244,7 +247,10 @@ func (o *OrService) getNgxServer(conf *v1.Config) (l7srv []*model.Server, l4srv "tenant_id": vs.Namespace, "service_id": vs.ServiceID, }, - UpstreamName: vs.PoolName, + UpstreamName: vs.PoolName, + ProxyStreamNextUpstream: true, + ProxyStreamNextUpstreamTimeout: "600s", + ProxyStreamNextUpstreamTries: 3, } server.Listen = strings.Join(vs.Listening, " ") l4srv = append(l4srv, server) diff --git a/gateway/controller/openresty/template/func_map.go b/gateway/controller/openresty/template/func_map.go index 9bef2293a..1196b0e70 100644 --- a/gateway/controller/openresty/template/func_map.go +++ b/gateway/controller/openresty/template/func_map.go @@ -30,6 +30,11 @@ import ( "github.com/sirupsen/logrus" ) +const ( + slash = "/" + nonIdempotent = "non_idempotent" +) + var ( funcMap = text_template.FuncMap{ "empty": func(input interface{}) bool { @@ -41,6 +46,7 @@ var ( }, "buildLuaHeaderRouter": buildLuaHeaderRouter, "isValidByteSize": isValidByteSize, + "buildNextUpstream": buildNextUpstream, } ) @@ -153,3 +159,32 @@ func isValidByteSize(input interface{}, isOffset bool) bool { return nginxSizeRegex.MatchString(s) } + +func buildNextUpstream(i, r interface{}) string { + nextUpstream, ok := i.(string) + if !ok { + logrus.Errorf("expected a 'string' type but %T was returned", i) + return "" + } + + retryNonIdempotent := r.(bool) + + parts := strings.Split(nextUpstream, " ") + + nextUpstreamCodes := make([]string, 0, len(parts)) + for _, v := range parts { + if v != "" && v != nonIdempotent { + nextUpstreamCodes = append(nextUpstreamCodes, v) + } + + if v == nonIdempotent { + retryNonIdempotent = true + } + } + + if retryNonIdempotent { + nextUpstreamCodes = append(nextUpstreamCodes, nonIdempotent) + } + + return strings.Join(nextUpstreamCodes, " ") +} diff --git a/gateway/defaults/defaults.go b/gateway/defaults/defaults.go index 6ff4fad9d..b89c42194 100644 --- a/gateway/defaults/defaults.go +++ b/gateway/defaults/defaults.go @@ -77,6 +77,8 @@ type Backend struct { // https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream_tries ProxyNextUpstreamTries int `json:"proxy-next-upstream-tries"` + ProxyNextUpstreamTimeout int `json:"proxy-next-upstream-timeout"` + // Sets the original text that should be changed in the "Location" and "Refresh" header fields of a proxied server response. // http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_redirect // Default: off diff --git a/hack/contrib/docker/gateway/nginx/lua/defaultPage.lua b/hack/contrib/docker/gateway/nginx/lua/defaultPage.lua index f9b53bb49..00787d88c 100644 --- a/hack/contrib/docker/gateway/nginx/lua/defaultPage.lua +++ b/hack/contrib/docker/gateway/nginx/lua/defaultPage.lua @@ -274,7 +274,6 @@ function _M.call() html = string.gsub(_M.defaultHTML, "POWER", [[

Power By Rainbond

]],1) end ngx.print(html) - ngx.status = ngx.HTTP_OK end return _M diff --git a/hack/contrib/docker/gateway/nginxtmp/servers.tmpl b/hack/contrib/docker/gateway/nginxtmp/servers.tmpl index bde72efa7..5396f56b7 100644 --- a/hack/contrib/docker/gateway/nginxtmp/servers.tmpl +++ b/hack/contrib/docker/gateway/nginxtmp/servers.tmpl @@ -68,6 +68,10 @@ server { proxy_send_timeout {{ $loc.Proxy.SendTimeout }}s; proxy_read_timeout {{ $loc.Proxy.ReadTimeout }}s; + proxy_next_upstream {{ buildNextUpstream $loc.Proxy.NextUpstream false }}; + proxy_next_upstream_timeout {{ $loc.Proxy.NextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $loc.Proxy.NextUpstreamTries }}; + proxy_buffering {{ $loc.Proxy.ProxyBuffering }}; proxy_buffer_size {{ $loc.Proxy.BufferSize }}; proxy_buffers {{ $loc.Proxy.BuffersNumber }} {{ $loc.Proxy.BufferSize }}; diff --git a/hack/contrib/docker/gateway/nginxtmp/tcp_udp_servers.tmpl b/hack/contrib/docker/gateway/nginxtmp/tcp_udp_servers.tmpl index 6f7e50568..53db554c0 100644 --- a/hack/contrib/docker/gateway/nginxtmp/tcp_udp_servers.tmpl +++ b/hack/contrib/docker/gateway/nginxtmp/tcp_udp_servers.tmpl @@ -8,6 +8,9 @@ server { {{ if .Listen }}listen {{.Listen}} {{ if $tcpServer.ProxyProtocol.Decode }} proxy_protocol{{ end }};{{ end }} proxy_timeout {{ $tcpServer.ProxyStreamTimeout }}; proxy_pass upstream_balancer; + proxy_next_upstream {{ if $tcpServer.ProxyStreamNextUpstream }}on{{ else }}off{{ end }}; + proxy_next_upstream_timeout {{ $tcpServer.ProxyStreamNextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $tcpServer.ProxyStreamNextUpstreamTries }}; {{ if $tcpServer.ProxyProtocol.Encode }} proxy_protocol on; {{ end }} @@ -23,6 +26,9 @@ server { {{ if $udpServer.Listen }}listen {{$udpServer.Listen}} {{ if $udpServer.ProxyProtocol.Decode }} proxy_protocol{{ end }};{{ end }} {{ if $udpServer.ProxyStreamResponses }}proxy_responses {{ $udpServer.ProxyStreamResponses }}; {{ end }} proxy_timeout {{ $udpServer.ProxyStreamTimeout }}; + proxy_next_upstream {{ if $udpServer.ProxyStreamNextUpstream }}on{{ else }}off{{ end }}; + proxy_next_upstream_timeout {{ $udpServer.ProxyStreamNextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $udpServer.ProxyStreamNextUpstreamTries }}; proxy_pass upstream_balancer; } {{ end }} \ No newline at end of file