add backend-shards command-line option

Big clusters waste some time and cpu executing the haproxy template every time a configuration changes, even if such configuration is dinamically applied, leaving memory and disk configurations in sync. Most of the time is spent rebuilding all the referenced services, which is converted to haproxy backends. Splitting the backends into smaller shards give haproxy ingress the ability to only update files that have changes to be applied, reducing io and cpu usage. The shard of a backend is chosen from the hash of its ID, so the same backend will always stay in the same shard provided that the number of shards isn't changed. Updating the number of shards will rebalance the backends.
jcmoraisjr · Jul 22, 2020 · e6ef0c2 · e6ef0c2
1 parent 8a95242
commit e6ef0c2
Show file tree

Hide file tree

Showing 14 changed files with 525 additions and 130 deletions.
diff --git a/docs/content/en/docs/configuration/command-line.md b/docs/content/en/docs/configuration/command-line.md
@@ -20,6 +20,7 @@ The following command-line options are supported:
 | [`--acme-track-tls-annotation`](#acme)                  | [true\|false]              | `false`                 | v0.9 |
 | [`--allow-cross-namespace`](#allow-cross-namespace)     | [true\|false]              | `false`                 |       |
 | [`--annotation-prefix`](#annotation-prefix)             | prefix without `/`         | `ingress.kubernetes.io` | v0.8  |
+| [`--backend-shards`](#backend-shards)                   | int                        | `0`                     | v0.11 |
 | [`--buckets-response-time`](#buckets-response-time)     | float64 slice           | `.0005,.001,.002,.005,.01` | v0.10 |
 | [`--default-backend-service`](#default-backend-service) | namespace/servicename      | haproxy's 404 page      |       |
 | [`--default-ssl-certificate`](#default-ssl-certificate) | namespace/secretname       | fake, auto generated    |       |
@@ -82,6 +83,16 @@ that shares ingress and service objects without conflicting each other.
 
 ---
 
+## --backend-shards
+
+Defines how much files should be used to configure the haproxy backends. The default value is
+0 (zero) which uses one single file to configure the whole haproxy process. Values greather than
+0 (zero) splits the backend configuration into separated files. Only files with changed backends
+are parsed and written to disk, reducing io and cpu usage on big clusters - about 1000 or more
+services.
+
+---
+
 ## --buckets-response-time
 
 Configures the buckets of the histogram `haproxyingress_haproxy_response_time_seconds`, used to compute the response time of the haproxy's admin socket. The response time unit is in seconds.

diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go
@@ -94,6 +94,7 @@ type Configuration struct {
 	ElectionID             string
 	UpdateStatusOnShutdown bool
 
+	BackendShards             int
 	SortBackends              bool
 	IgnoreIngressWithoutClass bool
 }

diff --git a/pkg/common/ingress/controller/launch.go b/pkg/common/ingress/controller/launch.go
@@ -152,6 +152,9 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 		ingress controller should update the Ingress status IP/hostname when the controller
 		is being stopped. Default is true`)
 
+		backendShards = flags.Int("backend-shards", 0,
+			`Defines how much files should be used to configure the haproxy backends`)
+
 		sortBackends = flags.Bool("sort-backends", false,
 			`Defines if backends and it's endpoints should be sorted`)
 
@@ -311,6 +314,7 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 		AllowCrossNamespace:       *allowCrossNamespace,
 		DisableNodeList:           *disableNodeList,
 		UpdateStatusOnShutdown:    *updateStatusOnShutdown,
+		BackendShards:             *backendShards,
 		SortBackends:              *sortBackends,
 		UseNodeInternalIP:         *useNodeInternalIP,
 		IgnoreIngressWithoutClass: *ignoreIngressWithoutClass,

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -120,7 +120,9 @@ func (hc *HAProxyController) configController() {
 	instanceOptions := haproxy.InstanceOptions{
 		HAProxyCmd:        "haproxy",
 		ReloadCmd:         "/haproxy-reload.sh",
-		HAProxyConfigFile: "/etc/haproxy/haproxy.cfg",
+		HAProxyCfgDir:     "/etc/haproxy",
+		HAProxyMapsDir:    "/etc/haproxy/maps",
+		BackendShards:     hc.cfg.BackendShards,
 		AcmeSigner:        acmeSigner,
 		AcmeQueue:         hc.acmeQueue,
 		LeaderElector:     hc.leaderelector,

diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go
@@ -45,9 +45,8 @@ type Config interface {
 
 type config struct {
 	// external state, non haproxy data
-	acmeData     *hatypes.AcmeData
-	mapsTemplate *template.Config
-	mapsDir      string
+	options  options
+	acmeData *hatypes.AcmeData
 	// haproxy internal state
 	globalOld   *hatypes.Global
 	global      *hatypes.Global
@@ -59,26 +58,24 @@ type config struct {
 }
 
 type options struct {
-	// reflect changes to config.Clear()
 	mapsTemplate *template.Config
 	mapsDir      string
+	shardCount   int
 }
 
 func createConfig(options options) *config {
-	mapsTemplate := options.mapsTemplate
-	if mapsTemplate == nil {
-		mapsTemplate = template.CreateConfig()
+	if options.mapsTemplate == nil {
+		options.mapsTemplate = template.CreateConfig()
 	}
 	return &config{
-		acmeData:     &hatypes.AcmeData{},
-		global:       &hatypes.Global{},
-		frontend:     &hatypes.Frontend{Name: "_front001"},
-		hosts:        hatypes.CreateHosts(),
-		backends:     hatypes.CreateBackends(),
-		tcpbackends:  hatypes.CreateTCPBackends(),
-		userlists:    hatypes.CreateUserlists(),
-		mapsTemplate: mapsTemplate,
-		mapsDir:      options.mapsDir,
+		options:     options,
+		acmeData:    &hatypes.AcmeData{},
+		global:      &hatypes.Global{},
+		frontend:    &hatypes.Frontend{Name: "_front001"},
+		hosts:       hatypes.CreateHosts(),
+		backends:    hatypes.CreateBackends(options.shardCount),
+		tcpbackends: hatypes.CreateTCPBackends(),
+		userlists:   hatypes.CreateUserlists(),
 	}
 }
 
@@ -143,23 +140,24 @@ func (c *config) SyncConfig() {
 // link to the frontend maps.
 func (c *config) WriteFrontendMaps() error {
 	mapBuilder := hatypes.CreateMaps()
+	mapsDir := c.options.mapsDir
 	fmaps := &hatypes.FrontendMaps{
-		HTTPFrontsMap:     mapBuilder.AddMap(c.mapsDir + "/_global_http_front.map"),
-		HTTPRootRedirMap:  mapBuilder.AddMap(c.mapsDir + "/_global_http_root_redir.map"),
-		HTTPSRedirMap:     mapBuilder.AddMap(c.mapsDir + "/_global_https_redir.map"),
-		SSLPassthroughMap: mapBuilder.AddMap(c.mapsDir + "/_global_sslpassthrough.map"),
-		VarNamespaceMap:   mapBuilder.AddMap(c.mapsDir + "/_global_k8s_ns.map"),
+		HTTPFrontsMap:     mapBuilder.AddMap(mapsDir + "/_global_http_front.map"),
+		HTTPRootRedirMap:  mapBuilder.AddMap(mapsDir + "/_global_http_root_redir.map"),
+		HTTPSRedirMap:     mapBuilder.AddMap(mapsDir + "/_global_https_redir.map"),
+		SSLPassthroughMap: mapBuilder.AddMap(mapsDir + "/_global_sslpassthrough.map"),
+		VarNamespaceMap:   mapBuilder.AddMap(mapsDir + "/_global_k8s_ns.map"),
 		//
-		HostBackendsMap:            mapBuilder.AddMap(c.mapsDir + "/_front001_host.map"),
-		RootRedirMap:               mapBuilder.AddMap(c.mapsDir + "/_front001_root_redir.map"),
-		SNIBackendsMap:             mapBuilder.AddMap(c.mapsDir + "/_front001_sni.map"),
-		TLSInvalidCrtErrorList:     mapBuilder.AddMap(c.mapsDir + "/_front001_inv_crt.list"),
-		TLSInvalidCrtErrorPagesMap: mapBuilder.AddMap(c.mapsDir + "/_front001_inv_crt_redir.map"),
-		TLSNoCrtErrorList:          mapBuilder.AddMap(c.mapsDir + "/_front001_no_crt.list"),
-		TLSNoCrtErrorPagesMap:      mapBuilder.AddMap(c.mapsDir + "/_front001_no_crt_redir.map"),
+		HostBackendsMap:            mapBuilder.AddMap(mapsDir + "/_front001_host.map"),
+		RootRedirMap:               mapBuilder.AddMap(mapsDir + "/_front001_root_redir.map"),
+		SNIBackendsMap:             mapBuilder.AddMap(mapsDir + "/_front001_sni.map"),
+		TLSInvalidCrtErrorList:     mapBuilder.AddMap(mapsDir + "/_front001_inv_crt.list"),
+		TLSInvalidCrtErrorPagesMap: mapBuilder.AddMap(mapsDir + "/_front001_inv_crt_redir.map"),
+		TLSNoCrtErrorList:          mapBuilder.AddMap(mapsDir + "/_front001_no_crt.list"),
+		TLSNoCrtErrorPagesMap:      mapBuilder.AddMap(mapsDir + "/_front001_no_crt_redir.map"),
 		//
-		CrtList:       mapBuilder.AddMap(c.mapsDir + "/_front001_bind_crt.list"),
-		UseServerList: mapBuilder.AddMap(c.mapsDir + "/_front001_use_server.list"),
+		CrtList:       mapBuilder.AddMap(mapsDir + "/_front001_bind_crt.list"),
+		UseServerList: mapBuilder.AddMap(mapsDir + "/_front001_use_server.list"),
 	}
 	fmaps.CrtList.AppendItem(c.frontend.DefaultCert)
 	// Some maps use yes/no answers instead of a list with found/missing keys
@@ -283,7 +281,7 @@ func (c *config) WriteFrontendMaps() error {
 			fmaps.CrtList.AppendItem(crtListEntry)
 		}
 	}
-	if err := writeMaps(mapBuilder, c.mapsTemplate); err != nil {
+	if err := writeMaps(mapBuilder, c.options.mapsTemplate); err != nil {
 		return err
 	}
 	c.frontend.Maps = fmaps
@@ -299,15 +297,15 @@ func (c *config) WriteBackendMaps() error {
 	mapBuilder := hatypes.CreateMaps()
 	for _, backend := range c.backends.Items() {
 		if backend.NeedACL() {
-			mapsPrefix := c.mapsDir + "/_back_" + backend.ID
+			mapsPrefix := c.options.mapsDir + "/_back_" + backend.ID
 			pathsMap := mapBuilder.AddMap(mapsPrefix + "_idpath.map")
 			for _, path := range backend.Paths {
 				pathsMap.AppendPath(path.Hostpath, path.ID)
 			}
 			backend.PathsMap = pathsMap
 		}
 	}
-	return writeMaps(mapBuilder, c.mapsTemplate)
+	return writeMaps(mapBuilder, c.options.mapsTemplate)
 }
 
 func writeMaps(maps *hatypes.HostsMaps, template *template.Config) error {
@@ -349,10 +347,7 @@ func (c *config) Userlists() *hatypes.Userlists {
 }
 
 func (c *config) Clear() {
-	config := createConfig(options{
-		mapsTemplate: c.mapsTemplate,
-		mapsDir:      c.mapsDir,
-	})
+	config := createConfig(c.options)
 	*c = *config
 }
 

diff --git a/pkg/haproxy/config_test.go b/pkg/haproxy/config_test.go
@@ -65,7 +65,7 @@ func TestClear(t *testing.T) {
 	})
 	c.Hosts().AcquireHost("app.local")
 	c.Backends().AcquireBackend("default", "app", "8080")
-	if c.mapsDir != "/tmp/maps" {
+	if c.options.mapsDir != "/tmp/maps" {
 		t.Error("expected mapsDir == /tmp/maps")
 	}
 	if len(c.Hosts().Items()) != 1 {
@@ -75,7 +75,7 @@ func TestClear(t *testing.T) {
 		t.Error("expected len(backends) == 1")
 	}
 	c.Clear()
-	if c.mapsDir != "/tmp/maps" {
+	if c.options.mapsDir != "/tmp/maps" {
 		t.Error("expected mapsDir == /tmp/maps")
 	}
 	if len(c.Hosts().Items()) != 0 {

diff --git a/pkg/haproxy/dynupdate_test.go b/pkg/haproxy/dynupdate_test.go
@@ -585,11 +585,10 @@ set server default_app_8080/srv002 weight 1`,
 	}
 	for i, test := range testCases {
 		c := setup(t)
-		instance := c.instance.(*instance)
 		if test.doconfig1 != nil {
 			test.doconfig1(c)
 		}
-		instance.config.Commit()
+		c.instance.config.Commit()
 		backendIDs := []types.BackendID{}
 		for _, backend := range c.config.Backends().Items() {
 			if backend != c.config.Backends().DefaultBackend() {
@@ -601,7 +600,7 @@ set server default_app_8080/srv002 weight 1`,
 			test.doconfig2(c)
 		}
 		var cmd string
-		dynUpdater := instance.newDynUpdater()
+		dynUpdater := c.instance.newDynUpdater()
 		dynUpdater.cmd = func(socket string, observer func(duration time.Duration), command ...string) ([]string, error) {
 			for _, c := range command {
 				cmd = cmd + c + "\n"