Connection limits and timeout config

README.md

@@ -67,6 +67,9 @@ The following annotations are supported:
 |`[0]`|[`ingress.kubernetes.io/limit-connections`](#limit)|qty|-|
 |`[0]`|[`ingress.kubernetes.io/limit-rps`](#limit)|rate per second|-|
 |`[0]`|[`ingress.kubernetes.io/limit-whitelist`](#limit)|cidr list|-|
+|`[1]`|[`ingress.kubernetes.io/maxconn-server`](#connection)|qty|-|
+|`[1]`|[`ingress.kubernetes.io/maxqueue-server`](#connection)|qty|-|
+|`[1]`|[`ingress.kubernetes.io/timeout-queue`](#connection)|qty|-|
 ||[`ingress.kubernetes.io/proxy-body-size`](#proxy-body-size)|size (bytes)|-|
 ||`ingress.kubernetes.io/secure-backends`|[true\|false]|-|
 ||`ingress.kubernetes.io/secure-verify-ca-secret`|secret name|-|
@@ -155,6 +158,19 @@ The following annotations are supported:
 * `ingress.kubernetes.io/limit-rps`: Maximum number of connections per second of the same IP
 * `ingress.kubernetes.io/limit-whitelist`: Comma separated list of CIDRs that should be removed from the rate limit and concurrent connections check
 
+### Connection
+
+Cconfigurations of connection limit and timeout.
+
+* `ingress.kubernetes.io/maxconn-server`: Defines the maximum concurrent connections each server of a backend should receive. If not specified or a value lesser than or equal zero is used, an unlimited number of connections will be allowed. When the limit is reached, new connections will wait on a queue.
+* `ingress.kubernetes.io/maxqueue-server`: Defines the maximum number of connections should wait in the queue of a server. When this number is reached, new requests will be redispached to another server, breaking sticky session if configured. The queue will be unlimited if the annotation is not specified or a value lesser than or equal zero is used.
+* `ingress.kubernetes.io/timeout-queue`: Defines how much time a connection should wait on a queue before a 503 error is returned to the client. The unit defaults to milliseconds if missing, change the unit with `s`, `m`, `h`, ... suffix. The configmap `timeout-queue` option is used as the default value.
+
+* http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#5.2-maxconn
+* http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#5.2-maxqueue
+* http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-timeout%20queue
+* Time suffix: http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#2.4
+
 ### Server Alias
 
 Creates an alias of the server that annotation belongs to.
@@ -227,6 +243,7 @@ The following parameters are supported:
 ||[`timeout-connect`](#timeout)|time with suffix|`5s`|
 ||[`timeout-http-request`](#timeout)|time with suffix|`5s`|
 ||[`timeout-keep-alive`](#timeout)|time with suffix|`1m`|
+||[`timeout-queue`](#timeout)|time with suffix|`5s`|
 ||[`timeout-server`](#timeout)|time with suffix|`50s`|
 ||[`timeout-server-fin`](#timeout)|time with suffix|`50s`|
 ||[`timeout-tunnel`](#timeout)|time with suffix|`1h`|
@@ -451,6 +468,7 @@ Define timeout configurations:
 * `timeout-connect`: Maximum time to wait for a connection to a backend
 * `timeout-http-request`: Maximum time to wait for a complete HTTP request
 * `timeout-keep-alive`: Maximum time to wait for a new HTTP request on keep-alive connections
+* `timeout-queue`: Maximum time a connection should wait on a server queue before return a 503 error to the client
 * `timeout-server`: Maximum inactivity time on the backend side
 * `timeout-server-fin`: Maximum inactivity time on the backend side for half-closed connections - FIN_WAIT state
 * `timeout-tunnel`: Maximum inactivity time on the client and backend side for tunnels

pkg/common/ingress/annotations/connection/main.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package connection
+
+import (
+	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/parser"
+	extensions "k8s.io/api/extensions/v1beta1"
+)
+
+const (
+	maxconnServerAnn  = "ingress.kubernetes.io/maxconn-server"
+	maxqueueServerAnn = "ingress.kubernetes.io/maxqueue-server"
+	timeoutQueueAnn   = "ingress.kubernetes.io/timeout-queue"
+)
+
+// Config is the connection configuration
+type Config struct {
+	MaxConnServer  int
+	MaxQueueServer int
+	TimeoutQueue   string
+}
+
+type conn struct {
+}
+
+// NewParser creates a new connection annotation parser
+func NewParser() parser.IngressAnnotation {
+	return conn{}
+}
+
+// Parse parses connection limits and timeouts annotations and creates a Config struct
+func (c conn) Parse(ing *extensions.Ingress) (interface{}, error) {
+	maxconn, err := parser.GetIntAnnotation(maxconnServerAnn, ing)
+	if err != nil {
+		maxconn = 0
+	}
+	maxqueue, err := parser.GetIntAnnotation(maxqueueServerAnn, ing)
+	if err != nil {
+		maxqueue = 0
+	}
+	timeoutqueue, err := parser.GetStringAnnotation(timeoutQueueAnn, ing)
+	if err != nil {
+		timeoutqueue = ""
+	}
+	return &Config{
+		MaxConnServer:  maxconn,
+		MaxQueueServer: maxqueue,
+		TimeoutQueue:   timeoutqueue,
+	}, nil
+}
+
+// Equal tests equality between two Config objects
+func (c1 *Config) Equal(c2 *Config) bool {
+	if c1.MaxConnServer != c2.MaxConnServer {
+		return false
+	}
+	if c1.MaxQueueServer != c2.MaxQueueServer {
+		return false
+	}
+	if c1.TimeoutQueue != c2.TimeoutQueue {
+		return false
+	}
+	return true
+}

pkg/common/ingress/controller/annotations.go

@@ -26,6 +26,7 @@ import (
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/balance"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/clientbodybuffersize"
+	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/connection"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/cors"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/defaultbackend"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/healthcheck"
@@ -79,6 +80,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor {
 			"UsePortInRedirects":   portinredirect.NewParser(cfg),
 			"Proxy":                proxy.NewParser(cfg),
 			"RateLimit":            ratelimit.NewParser(cfg),
+			"Connection":           connection.NewParser(),
 			"Redirect":             redirect.NewParser(),
 			"Rewrite":              rewrite.NewParser(cfg),
 			"SecureUpstream":       secureupstream.NewParser(cfg, cfg),
@@ -139,6 +141,7 @@ const (
 	sslPassthrough       = "SSLPassthrough"
 	sessionAffinity      = "SessionAffinity"
 	serviceUpstream      = "ServiceUpstream"
+	conn                 = "Connection"
 	serverAlias          = "Alias"
 	corsConfig           = "CorsConfig"
 	clientBodyBufferSize = "ClientBodyBufferSize"
@@ -166,6 +169,11 @@ func (e *annotationExtractor) SecureUpstream(ing *extensions.Ingress) *secureups
 	return secure
 }
 
+func (e *annotationExtractor) Connection(ing *extensions.Ingress) *connection.Config {
+	val, _ := e.annotations[conn].Parse(ing)
+	return val.(*connection.Config)
+}
+
 func (e *annotationExtractor) HealthCheck(ing *extensions.Ingress) *healthcheck.Upstream {
 	val, _ := e.annotations[healthCheck].Parse(ing)
 	return val.(*healthcheck.Upstream)

pkg/common/ingress/controller/controller.go

@@ -497,6 +497,7 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress)
 		balance := ic.annotations.BalanceAlgorithm(ing)
 		blueGreen := ic.annotations.BlueGreen(ing)
 		anns := ic.annotations.Extract(ing)
+		conn := ic.annotations.Connection(ing)
 
 		for _, rule := range ing.Spec.Rules {
 			host := rule.Host
@@ -607,6 +608,16 @@ func (ic *GenericController) getBackendServers(ingresses []*extensions.Ingress)
 				if len(ups.BlueGreen.DeployWeight) == 0 {
 					ups.BlueGreen = *blueGreen
 				}
+
+				if ups.Connection.MaxConnServer == 0 {
+					ups.Connection.MaxConnServer = conn.MaxConnServer
+				}
+				if ups.Connection.MaxQueueServer == 0 {
+					ups.Connection.MaxQueueServer = conn.MaxQueueServer
+				}
+				if ups.Connection.TimeoutQueue == "" {
+					ups.Connection.TimeoutQueue = conn.TimeoutQueue
+				}
 			}
 		}
 	}

pkg/common/ingress/types.go

@@ -32,6 +32,7 @@ import (
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/authreq"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/authtls"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/bluegreen"
+	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/connection"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/cors"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/ipwhitelist"
 	"github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/annotations/proxy"
@@ -193,6 +194,8 @@ type Backend struct {
 	BalanceAlgorithm string `json:"balanceAlgorithm"`
 	// BlueGreen has the blue/green deployment configuration
 	BlueGreen bluegreen.Config `json:"blueGreen"`
+	// Connection has backend or server connection limits and timeouts
+	Connection connection.Config `json:"connection"`
 	// Consistent hashing by NGINX variable
 	UpstreamHashBy string `json:"upstream-hash-by,omitempty"`
 }

pkg/common/ingress/types_equals.go

@@ -185,6 +185,10 @@ func (b1 *Backend) Equal(b2 *Backend) bool {
 		return false
 	}
 
+	if !b1.Connection.Equal(&b2.Connection) {
+		return false
+	}
+
 	for _, udp1 := range b1.Endpoints {
 		found := false
 		for _, udp2 := range b2.Endpoints {

pkg/controller/config.go

@@ -93,6 +93,7 @@ func newHAProxyConfig(haproxyController *HAProxyController) *types.HAProxyConfig
 		TimeoutConnect:              "5s",
 		TimeoutClient:               "50s",
 		TimeoutClientFin:            "50s",
+		TimeoutQueue:                "5s",
 		TimeoutServer:               "50s",
 		TimeoutServerFin:            "50s",
 		TimeoutTunnel:               "1h",

pkg/types/types.go

@@ -54,6 +54,7 @@ type (
 		TimeoutClient               string `json:"timeout-client"`
 		TimeoutClientFin            string `json:"timeout-client-fin"`
 		TimeoutServer               string `json:"timeout-server"`
+		TimeoutQueue                string `json:"timeout-queue"`
 		TimeoutServerFin            string `json:"timeout-server-fin"`
 		TimeoutTunnel               string `json:"timeout-tunnel"`
 		TimeoutKeepAlive            string `json:"timeout-keep-alive"`

rootfs/etc/haproxy/template/haproxy.tmpl

@@ -39,6 +39,7 @@ defaults
     timeout connect         {{ $cfg.TimeoutConnect }}
     timeout client          {{ $cfg.TimeoutClient }}
     timeout client-fin      {{ $cfg.TimeoutClientFin }}
+    timeout queue           {{ $cfg.TimeoutQueue }}
     timeout server          {{ $cfg.TimeoutServer }}
     timeout server-fin      {{ $cfg.TimeoutServerFin }}
     timeout tunnel          {{ $cfg.TimeoutTunnel }}
@@ -88,6 +89,9 @@ listen tcp-{{ $tcp.Port }}
 backend {{ $backend.Name }}
     mode {{ if $backend.SSLPassthrough }}tcp{{ else }}http{{ end }}
     balance {{ $backend.BalanceAlgorithm }}
+{{- if ne $backend.Connection.TimeoutQueue "" }}
+    timeout queue {{ $backend.Connection.TimeoutQueue }}
+{{- end }}
 {{- $sticky := $backend.SessionAffinity }}
 {{- if eq $sticky.AffinityType "cookie" }}
     cookie {{ $sticky.CookieSessionAffinity.Name }} {{ $sticky.CookieSessionAffinity.Strategy }} {{ if eq $sticky.CookieSessionAffinity.Strategy "insert" }}indirect nocache{{ end }} dynamic
@@ -99,10 +103,10 @@ backend {{ $backend.Name }}
 {{- end }}
 {{- $BackendSlots := index $ing.BackendSlots $backend.Name }}
 {{- range $target, $slot := $BackendSlots.FullSlots }}
-    server {{ $slot.BackendServerName }} {{ $target }} {{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}{{ if ge $slot.BackendEndpoint.Weight 0 }}weight {{ $slot.BackendEndpoint.Weight }} {{ end }}check port {{ $slot.BackendEndpoint.Port }} inter {{ $cfg.BackendCheckInterval }}
+    server {{ $slot.BackendServerName }} {{ $target }} {{ if gt $backend.Connection.MaxConnServer 0 }}maxconn {{ $backend.Connection.MaxConnServer }} {{ end }}{{ if gt $backend.Connection.MaxQueueServer 0 }}maxqueue {{ $backend.Connection.MaxQueueServer }} {{ end }}{{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}{{ if ge $slot.BackendEndpoint.Weight 0 }}weight {{ $slot.BackendEndpoint.Weight }} {{ end }}check port {{ $slot.BackendEndpoint.Port }} inter {{ $cfg.BackendCheckInterval }}
 {{- end }}
 {{- range $empty := $BackendSlots.EmptySlots }}
-    server {{ $empty }} 127.0.0.1:81 {{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}check disabled inter {{ $cfg.BackendCheckInterval }}
+    server {{ $empty }} 127.0.0.1:81 {{ if gt $backend.Connection.MaxConnServer 0 }}maxconn {{ $backend.Connection.MaxConnServer }} {{ end }}{{ if gt $backend.Connection.MaxQueueServer 0 }}maxqueue {{ $backend.Connection.MaxQueueServer }} {{ end }}{{ if $backend.Secure }}ssl {{ if ne $cacert.CAFileName "" }}verify required ca-file {{ $cacert.CAFileName }} {{ else }}verify none {{ end }}{{ end }}check disabled inter {{ $cfg.BackendCheckInterval }}
 {{- end }}
 {{- end }}{{/* range Backends */}}