From 4a34a42b06c1b9544ed59161646af95f447f93c0 Mon Sep 17 00:00:00 2001 From: Joao Morais <jcmoraisjr@gmail.com> Date: Sun, 6 May 2018 11:32:40 -0300 Subject: [PATCH] Connection limits and timeout config --- README.md | 18 +++++ .../ingress/annotations/connection/main.go | 78 +++++++++++++++++++ pkg/common/ingress/controller/annotations.go | 8 ++ pkg/common/ingress/controller/controller.go | 11 +++ pkg/common/ingress/types.go | 3 + pkg/common/ingress/types_equals.go | 4 + pkg/controller/config.go | 1 + pkg/types/types.go | 1 + rootfs/etc/haproxy/template/haproxy.tmpl | 8 +- 9 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 pkg/common/ingress/annotations/connection/main.go diff --git a/README.md b/README.md index 0fb529316..d6ea8082f 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,9 @@ The following annotations are supported: |`[0]`|[`ingress.kubernetes.io/limit-connections`](#limit)|qty|-| |`[0]`|[`ingress.kubernetes.io/limit-rps`](#limit)|rate per second|-| |`[0]`|[`ingress.kubernetes.io/limit-whitelist`](#limit)|cidr list|-| +|`[1]`|[`ingress.kubernetes.io/maxconn-server`](#connection)|qty|-| +|`[1]`|[`ingress.kubernetes.io/maxqueue-server`](#connection)|qty|-| +|`[1]`|[`ingress.kubernetes.io/timeout-queue`](#connection)|qty|-| ||[`ingress.kubernetes.io/proxy-body-size`](#proxy-body-size)|size (bytes)|-| ||`ingress.kubernetes.io/secure-backends`|[true\|false]|-| ||`ingress.kubernetes.io/secure-verify-ca-secret`|secret name|-| @@ -155,6 +158,19 @@ The following annotations are supported: * `ingress.kubernetes.io/limit-rps`: Maximum number of connections per second of the same IP * `ingress.kubernetes.io/limit-whitelist`: Comma separated list of CIDRs that should be removed from the rate limit and concurrent connections check +### Connection + +Cconfigurations of connection limit and timeout. + +* `ingress.kubernetes.io/maxconn-server`: Defines the maximum concurrent connections each server of a backend should receive. If not specified or a value lesser than or equal zero is used, an unlimited number of connections will be allowed. When the limit is reached, new connections will wait on a queue. +* `ingress.kubernetes.io/maxqueue-server`: Defines the maximum number of connections should wait in the queue of a server. When this number is reached, new requests will be redispached to another server, breaking sticky session if configured. The queue will be unlimited if the annotation is not specified or a value lesser than or equal zero is used. +* `ingress.kubernetes.io/timeout-queue`: Defines how much time a connection should wait on a queue before a 503 error is returned to the client. The unit defaults to milliseconds if missing, change the unit with `s`, `m`, `h`, ... suffix. The configmap `timeout-queue` option is used as the default value. + +* http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#5.2-maxconn +* http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#5.2-maxqueue +* http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-timeout%20queue +* Time suffix: http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#2.4 + ### Server Alias Creates an alias of the server that annotation belongs to. @@ -227,6 +243,7 @@ The following parameters are supported: ||[`timeout-connect`](#timeout)|time with suffix|`5s`| ||[`timeout-http-request`](#timeout)|time with suffix|`5s`| ||[`timeout-keep-alive`](#timeout)|time with suffix|`1m`| +||[`timeout-queue`](#timeout)|time with suffix|`5s`| ||[`timeout-server`](#timeout)|time with suffix|`50s`| ||[`timeout-server-fin`](#timeout)|time with suffix|`50s`| ||[`timeout-tunnel`](#timeout)|time with suffix|`1h`| @@ -451,6 +468,7 @@ Define timeout configurations: * `timeout-connect`: Maximum time to wait for a connection to a backend * `timeout-http-request`: Maximum time to wait for a complete HTTP request * `timeout-keep-alive`: Maximum time to wait for a new HTTP request on keep-alive connections +* `timeout-queue`: Maximum time a connection should wait on a server queue before return a 503 error to the client * `timeout-server`: Maximum inactivity time on the backend side * `timeout-server-fin`: Maximum inactivity time on the backend side for half-closed connections - FIN_WAIT state * `timeout-tunnel`: Maximum inactivity time on the client and backend side for tunnels diff --git a/pkg/common/ingress/annotations/connection/main.go b/pkg/common/ingress/annotations/connection/main.go new file mode 100644 index 000000000..358486ca0 --- /dev/null +++ b/pkg/common/ingress/annotations/connection/main.go @@ -0,0 +1,78 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package connection + +import ( + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/parser" + extensions "k8s.io/api/extensions/v1beta1" +) + +const ( + maxconnServerAnn = "ingress.kubernetes.io/maxconn-server" + maxqueueServerAnn = "ingress.kubernetes.io/maxqueue-server" + timeoutQueueAnn = "ingress.kubernetes.io/timeout-queue" +) + +// Config is the connection configuration +type Config struct { + MaxConnServer int + MaxQueueServer int + TimeoutQueue string +} + +type conn struct { +} + +// NewParser creates a new connection annotation parser +func NewParser() parser.IngressAnnotation { + return conn{} +} + +// Parse parses connection limits and timeouts annotations and creates a Config struct +func (c conn) Parse(ing *extensions.Ingress) (interface{}, error) { + maxconn, err := parser.GetIntAnnotation(maxconnServerAnn, ing) + if err != nil { + maxconn = 0 + } + maxqueue, err := parser.GetIntAnnotation(maxqueueServerAnn, ing) + if err != nil { + maxqueue = 0 + } + timeoutqueue, err := parser.GetStringAnnotation(timeoutQueueAnn, ing) + if err != nil { + timeoutqueue = "" + } + return &Config{ + MaxConnServer: maxconn, + MaxQueueServer: maxqueue, + TimeoutQueue: timeoutqueue, + }, nil +} + +// Equal tests equality between two Config objects +func (c1 *Config) Equal(c2 *Config) bool { + if c1.MaxConnServer != c2.MaxConnServer { + return false + } + if c1.MaxQueueServer != c2.MaxQueueServer { + return false + } + if c1.TimeoutQueue != c2.TimeoutQueue { + return false + } + return true +} diff --git a/pkg/common/ingress/controller/annotations.go b/pkg/common/ingress/controller/annotations.go index fc19c9412..aa4fb0545 100644 --- a/pkg/common/ingress/controller/annotations.go +++ b/pkg/common/ingress/controller/annotations.go @@ -26,6 +26,7 @@ import ( "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/balance" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/clientbodybuffersize" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/connection" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/cors" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/defaultbackend" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/healthcheck" @@ -79,6 +80,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor { "UsePortInRedirects": portinredirect.NewParser(cfg), "Proxy": proxy.NewParser(cfg), "RateLimit": ratelimit.NewParser(cfg), + "Connection": connection.NewParser(), "Redirect": redirect.NewParser(), "Rewrite": rewrite.NewParser(cfg), "SecureUpstream": secureupstream.NewParser(cfg, cfg), @@ -139,6 +141,7 @@ const ( sslPassthrough = "SSLPassthrough" sessionAffinity = "SessionAffinity" serviceUpstream = "ServiceUpstream" + conn = "Connection" serverAlias = "Alias" corsConfig = "CorsConfig" clientBodyBufferSize = "ClientBodyBufferSize" @@ -166,6 +169,11 @@ func (e *annotationExtractor) SecureUpstream(ing *extensions.Ingress) *secureups return secure } +func (e *annotationExtractor) Connection(ing *extensions.Ingress) *connection.Config { + val, _ := e.annotations[conn].Parse(ing) + return val.(*connection.Config) +} + func (e *annotationExtractor) HealthCheck(ing *extensions.Ingress) *healthcheck.Upstream { val, _ := e.annotations[healthCheck].Parse(ing) return val.(*healthcheck.Upstream) diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index 079744bb3..85f6cd55f 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -497,6 +497,7 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) balance := ic.annotations.BalanceAlgorithm(ing) blueGreen := ic.annotations.BlueGreen(ing) anns := ic.annotations.Extract(ing) + conn := ic.annotations.Connection(ing) for _, rule := range ing.Spec.Rules { host := rule.Host @@ -607,6 +608,16 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress) if len(ups.BlueGreen.DeployWeight) == 0 { ups.BlueGreen = *blueGreen } + + if ups.Connection.MaxConnServer == 0 { + ups.Connection.MaxConnServer = conn.MaxConnServer + } + if ups.Connection.MaxQueueServer == 0 { + ups.Connection.MaxQueueServer = conn.MaxQueueServer + } + if ups.Connection.TimeoutQueue == "" { + ups.Connection.TimeoutQueue = conn.TimeoutQueue + } } } } diff --git a/pkg/common/ingress/types.go b/pkg/common/ingress/types.go index f0c140c43..9c301e767 100644 --- a/pkg/common/ingress/types.go +++ b/pkg/common/ingress/types.go @@ -32,6 +32,7 @@ import ( "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/authreq" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/authtls" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/connection" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/cors" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/ipwhitelist" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/proxy" @@ -193,6 +194,8 @@ type Backend struct { BalanceAlgorithm string `json:"balanceAlgorithm"` // BlueGreen has the blue/green deployment configuration BlueGreen bluegreen.Config `json:"blueGreen"` + // Connection has backend or server connection limits and timeouts + Connection connection.Config `json:"connection"` // Consistent hashing by NGINX variable UpstreamHashBy string `json:"upstream-hash-by,omitempty"` } diff --git a/pkg/common/ingress/types_equals.go b/pkg/common/ingress/types_equals.go index 92b3c349f..da4c0b788 100644 --- a/pkg/common/ingress/types_equals.go +++ b/pkg/common/ingress/types_equals.go @@ -185,6 +185,10 @@ func (b1 *Backend) Equal(b2 *Backend) bool { return false } + if !b1.Connection.Equal(&b2.Connection) { + return false + } + for _, udp1 := range b1.Endpoints { found := false for _, udp2 := range b2.Endpoints { diff --git a/pkg/controller/config.go b/pkg/controller/config.go index f08dbd2b6..066b62055 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -93,6 +93,7 @@ func newHAProxyConfig(haproxyController *HAProxyController) *types.HAProxyConfig TimeoutConnect: "5s", TimeoutClient: "50s", TimeoutClientFin: "50s", + TimeoutQueue: "5s", TimeoutServer: "50s", TimeoutServerFin: "50s", TimeoutTunnel: "1h", diff --git a/pkg/types/types.go b/pkg/types/types.go index 98ee29920..77181ffa8 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -54,6 +54,7 @@ type ( TimeoutClient string `json:"timeout-client"` TimeoutClientFin string `json:"timeout-client-fin"` TimeoutServer string `json:"timeout-server"` + TimeoutQueue string `json:"timeout-queue"` TimeoutServerFin string `json:"timeout-server-fin"` TimeoutTunnel string `json:"timeout-tunnel"` TimeoutKeepAlive string `json:"timeout-keep-alive"` diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index 672b99824..d3ab0cb6e 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -39,6 +39,7 @@ defaults timeout connect {{ $cfg.TimeoutConnect }} timeout client {{ $cfg.TimeoutClient }} timeout client-fin {{ $cfg.TimeoutClientFin }} + timeout queue {{ $cfg.TimeoutQueue }} timeout server {{ $cfg.TimeoutServer }} timeout server-fin {{ $cfg.TimeoutServerFin }} timeout tunnel {{ $cfg.TimeoutTunnel }} @@ -88,6 +89,9 @@ listen tcp-{{ $tcp.Port }} backend {{ $backend.Name }} mode {{ if $backend.SSLPassthrough }}tcp{{ else }}http{{ end }} balance {{ $backend.BalanceAlgorithm }} +{{- if ne $backend.Connection.TimeoutQueue "" }} + timeout queue {{ $backend.Connection.TimeoutQueue }} +{{- end }} {{- $sticky := $backend.SessionAffinity }} {{- if eq $sticky.AffinityType "cookie" }} cookie {{ $sticky.CookieSessionAffinity.Name }} {{ $sticky.CookieSessionAffinity.Strategy }} {{ if eq $sticky.CookieSessionAffinity.Strategy "insert" }}indirect nocache{{ end }} dynamic @@ -99,10 +103,10 @@ backend {{ $backend.Name }} {{- end }} {{- $BackendSlots := index $ing.BackendSlots $backend.Name }} {{- range $target, $slot := $BackendSlots.FullSlots }} - server {{ $slot.BackendServerName }} {{ $target }} {{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}{{ if ge $slot.BackendEndpoint.Weight 0 }}weight {{ $slot.BackendEndpoint.Weight }} {{ end }}check port {{ $slot.BackendEndpoint.Port }} inter {{ $cfg.BackendCheckInterval }} + server {{ $slot.BackendServerName }} {{ $target }} {{ if gt $backend.Connection.MaxConnServer 0 }}maxconn {{ $backend.Connection.MaxConnServer }} {{ end }}{{ if gt $backend.Connection.MaxQueueServer 0 }}maxqueue {{ $backend.Connection.MaxQueueServer }} {{ end }}{{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}{{ if ge $slot.BackendEndpoint.Weight 0 }}weight {{ $slot.BackendEndpoint.Weight }} {{ end }}check port {{ $slot.BackendEndpoint.Port }} inter {{ $cfg.BackendCheckInterval }} {{- end }} {{- range $empty := $BackendSlots.EmptySlots }} - server {{ $empty }} 127.0.0.1:81 {{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}check disabled inter {{ $cfg.BackendCheckInterval }} + server {{ $empty }} 127.0.0.1:81 {{ if gt $backend.Connection.MaxConnServer 0 }}maxconn {{ $backend.Connection.MaxConnServer }} {{ end }}{{ if gt $backend.Connection.MaxQueueServer 0 }}maxqueue {{ $backend.Connection.MaxQueueServer }} {{ end }}{{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}check disabled inter {{ $cfg.BackendCheckInterval }} {{- end }} {{- end }}{{/* range Backends */}}