From 103318cca5234792269a31d8e3ddefaf87b60bd9 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sat, 17 Oct 2020 15:33:52 -0300 Subject: [PATCH] add --sort-endpoints-by command-line option Add a few more options on how to sort endpoints of a backend. Currently `--sort-backends` switches between the same k8s endpoints order and naming order - which only makes sense if `backend-server-naming` was changed. `--sort-endpoints-by` also brings back the old behavior (v0.7 and older) of missing `--sort-backends`: `random` option will shuffle endpoints of all backends whenever a haproxy reload is fired. --- .../en/docs/configuration/command-line.md | 18 ++++++++++++- pkg/common/ingress/controller/controller.go | 2 +- pkg/common/ingress/controller/launch.go | 21 ++++++++++++++-- pkg/controller/controller.go | 2 +- pkg/haproxy/instance.go | 10 +++++--- pkg/haproxy/types/backend.go | 25 ++++++++++++++++--- pkg/haproxy/types/backends.go | 11 ++++++-- 7 files changed, 76 insertions(+), 13 deletions(-) diff --git a/docs/content/en/docs/configuration/command-line.md b/docs/content/en/docs/configuration/command-line.md index acb6c78ba..7621da02b 100644 --- a/docs/content/en/docs/configuration/command-line.md +++ b/docs/content/en/docs/configuration/command-line.md @@ -36,6 +36,7 @@ The following command-line options are supported: | [`--rate-limit-update`](#rate-limit-update) | uploads per second (float) | `0.5` | | | [`--reload-strategy`](#reload-strategy) | [native\|reusesocket] | `reusesocket` | | | [`--sort-backends`](#sort-backends) | [true\|false] | `false` | | +| [`--sort-endpoints-by`](#sort-endpoints-by) | [endpoint\|ip\|name\|random] | `endpoint` | v0.11 | | [`--stats-collect-processing-period`](#stats) | time | `500ms` | v0.10 | | [`--tcp-services-configmap`](#tcp-services-configmap) | namespace/configmapname | no tcp svc | | | [`--verify-hostname`](#verify-hostname) | [true\|false] | `true` | | @@ -235,7 +236,8 @@ describes how it works. ## --sort-backends Defines if backend's endpoints should be sorted by name. Since v0.8 the endpoints will stay in the -same order found in the Kubernetes' endpoint objects if `--sort-backends` is missing. +same order found in the Kubernetes' endpoint objects if `--sort-backends` is missing. This option +has less precedence than `--sort-endpoints-by` if both are declared. In v0.7 and older version, if `--sort-backends` is missing, HAProxy Ingress randomly shuffle endpoints on each reload in order to avoid requesting always the same backends just after haproxy reloads. @@ -246,6 +248,20 @@ option, because the default value builds the server name using a numeric sequenc See also: * [backend-server-naming]({{% relref "keys#backend-server-naming" %}}) configuration key +* [sort-endpoints-by]({{% relref "#sort-endpoints-by" %}}) command-line option + +--- + +## --sort-endpoints-by + +Since v0.11 + +Defines in which order the endpoints of a backend should be sorted. + +* `endpoint`: this is the default value, uses the same order declared in the Kubernetes' Endpoint objects. `ep` is an alias to `endpoint` +* `ip`: sort endpoints by the IP and port of the destination server +* `name`: sort the endpoints by the name given to the server, see also [backend-server-naming]({{% relref "keys#backend-server-naming" %}}) +* `random`: randomly shuffle the endpoints every time haproxy needs to be reloaded, this option avoids to always send requests to the same endpoints depending on the balancing algorithm --- diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index 3dde9b984..c78dd6a11 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -98,7 +98,7 @@ type Configuration struct { UpdateStatusOnShutdown bool BackendShards int - SortBackends bool + SortEndpointsBy string IgnoreIngressWithoutClass bool } diff --git a/pkg/common/ingress/controller/launch.go b/pkg/common/ingress/controller/launch.go index d49abbe6a..5f6f8dea1 100644 --- a/pkg/common/ingress/controller/launch.go +++ b/pkg/common/ingress/controller/launch.go @@ -169,7 +169,12 @@ func NewIngressController(backend ingress.Controller) *GenericController { `Defines how much files should be used to configure the haproxy backends`) sortBackends = flags.Bool("sort-backends", false, - `Defines if backend's endpoints should be sorted by name. It uses the same k8s endpoint order if missing`) + `Defines if backend's endpoints should be sorted by name. This option has less precedence than + --sort-endpoints-by if both are declared.`) + + sortEndpointsBy = flags.String("sort-endpoints-by", "", + `Defines how to sort backend's endpoints. Allowed values are: 'endpoint' - same k8s endpoint order (default); + 'name' - server/endpoint name; 'ip' - server/endpoint IP and port; 'random' - shuffle endpoints on every haproxy reload`) useNodeInternalIP = flags.Bool("report-node-internal-ip-address", false, `Defines if the nodes IP address to be returned in the ingress status should be the internal instead of the external IP address`) @@ -296,6 +301,18 @@ func NewIngressController(backend ingress.Controller) *GenericController { glog.Fatal("Cannot use --allow-cross-namespace if --force-namespace-isolation is true") } + sortEndpoints := strings.ToLower(*sortEndpointsBy) + if sortEndpoints == "" { + if *sortBackends { + sortEndpoints = "name" + } else { + sortEndpoints = "endpoint" + } + } + if !stringInSlice(sortEndpoints, []string{"ep", "endpoint", "ip", "name", "random"}) { + glog.Fatalf("Unsupported --sort-endpoint-by option: %s", sortEndpoints) + } + config := &Configuration{ UpdateStatus: *updateStatus, ElectionID: *electionID, @@ -332,7 +349,7 @@ func NewIngressController(backend ingress.Controller) *GenericController { DisablePodList: *disablePodList, UpdateStatusOnShutdown: *updateStatusOnShutdown, BackendShards: *backendShards, - SortBackends: *sortBackends, + SortEndpointsBy: sortEndpoints, UseNodeInternalIP: *useNodeInternalIP, IgnoreIngressWithoutClass: *ignoreIngressWithoutClass, } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 5ff47a0a8..395cd13e6 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -130,7 +130,7 @@ func (hc *HAProxyController) configController() { Metrics: hc.metrics, ReloadStrategy: *hc.reloadStrategy, MaxOldConfigFiles: *hc.maxOldConfigFiles, - SortBackends: hc.cfg.SortBackends, + SortEndpointsBy: hc.cfg.SortEndpointsBy, ValidateConfig: *hc.validateConfig, } hc.instance = haproxy.CreateInstance(hc.logger, instanceOptions) diff --git a/pkg/haproxy/instance.go b/pkg/haproxy/instance.go index 94c0da31d..059981aef 100644 --- a/pkg/haproxy/instance.go +++ b/pkg/haproxy/instance.go @@ -44,7 +44,7 @@ type InstanceOptions struct { MaxOldConfigFiles int Metrics types.Metrics ReloadStrategy string - SortBackends bool + SortEndpointsBy string ValidateConfig bool // TODO Fake is used to skip real haproxy calls. Use a mock instead. fake bool @@ -258,8 +258,12 @@ func (i *instance) haproxyUpdate(timer *utils.Timer) { } updater := i.newDynUpdater() updated := updater.update() - if i.options.SortBackends { - i.config.Backends().SortChangedEndpoints() + if i.options.SortEndpointsBy != "random" { + i.config.Backends().SortChangedEndpoints(i.options.SortEndpointsBy) + } else if !updated { + // Only shuffle if need to reload + i.config.Backends().ShuffleAllEndpoints() + timer.Tick("shuffle_endpoints") } if !updated || updater.cmdCnt > 0 { // only need to rewrtite config files if: diff --git a/pkg/haproxy/types/backend.go b/pkg/haproxy/types/backend.go index c6b1354d7..eabcee9fb 100644 --- a/pkg/haproxy/types/backend.go +++ b/pkg/haproxy/types/backend.go @@ -18,9 +18,11 @@ package types import ( "fmt" + "math/rand" "reflect" "sort" "strings" + "time" ) // BackendID ... @@ -92,9 +94,26 @@ func (b *Backend) addEndpoint(ip string, port int, targetRef string) *Endpoint { return endpoint } -func (b *Backend) sortEndpoints() { - sort.SliceStable(b.Endpoints, func(i, j int) bool { - return b.Endpoints[i].Name < b.Endpoints[j].Name +func (b *Backend) sortEndpoints(sortBy string) { + ep := b.Endpoints + switch sortBy { + // ignoring "ep"/"endpoint" (use the k8s order) and "random" (shuffleEndpoints implements) + case "name": + sort.Slice(ep, func(i, j int) bool { + return ep[i].Name < ep[j].Name + }) + case "ip": + sort.Slice(ep, func(i, j int) bool { + return ep[i].IP < ep[j].IP + }) + } +} + +func (b *Backend) shuffleEndpoints() { + ep := b.Endpoints + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(ep), func(i, j int) { + ep[i], ep[j] = ep[j], ep[i] }) } diff --git a/pkg/haproxy/types/backends.go b/pkg/haproxy/types/backends.go index 36049d94b..c7bbac9a2 100644 --- a/pkg/haproxy/types/backends.go +++ b/pkg/haproxy/types/backends.go @@ -146,9 +146,16 @@ func (b *Backends) ChangedShards() []int { } // SortChangedEndpoints ... -func (b *Backends) SortChangedEndpoints() { +func (b *Backends) SortChangedEndpoints(sortBy string) { for _, backend := range b.itemsAdd { - backend.sortEndpoints() + backend.sortEndpoints(sortBy) + } +} + +// ShuffleAllEndpoints ... +func (b *Backends) ShuffleAllEndpoints() { + for _, backend := range b.items { + backend.shuffleEndpoints() } }