Merge pull request #623 from jcmoraisjr/jm-backend-shards

add backend-shards command-line option
jcmoraisjr · Jul 22, 2020 · 552e8fe · 552e8fe
2 parents 8a95242 + e6ef0c2
commit 552e8fe
Show file tree

Hide file tree

Showing 14 changed files with 525 additions and 130 deletions.
diff --git a/docs/content/en/docs/configuration/command-line.md b/docs/content/en/docs/configuration/command-line.md
@@ -20,6 +20,7 @@ The following command-line options are supported:
 | [`--acme-track-tls-annotation`](#acme)                  | [true\|false]              | `false`                 | v0.9 |
 | [`--allow-cross-namespace`](#allow-cross-namespace)     | [true\|false]              | `false`                 |       |
 | [`--annotation-prefix`](#annotation-prefix)             | prefix without `/`         | `ingress.kubernetes.io` | v0.8  |
+| [`--backend-shards`](#backend-shards)                   | int                        | `0`                     | v0.11 |
 | [`--buckets-response-time`](#buckets-response-time)     | float64 slice           | `.0005,.001,.002,.005,.01` | v0.10 |
 | [`--default-backend-service`](#default-backend-service) | namespace/servicename      | haproxy's 404 page      |       |
 | [`--default-ssl-certificate`](#default-ssl-certificate) | namespace/secretname       | fake, auto generated    |       |
@@ -82,6 +83,16 @@ that shares ingress and service objects without conflicting each other.
 
 ---
 
+## --backend-shards
+
+Defines how much files should be used to configure the haproxy backends. The default value is
+0 (zero) which uses one single file to configure the whole haproxy process. Values greather than
+0 (zero) splits the backend configuration into separated files. Only files with changed backends
+are parsed and written to disk, reducing io and cpu usage on big clusters - about 1000 or more
+services.
+
+---
+
 ## --buckets-response-time
 
 Configures the buckets of the histogram `haproxyingress_haproxy_response_time_seconds`, used to compute the response time of the haproxy's admin socket. The response time unit is in seconds.

diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go
@@ -94,6 +94,7 @@ type Configuration struct {
 	ElectionID             string
 	UpdateStatusOnShutdown bool
 
+	BackendShards             int
 	SortBackends              bool
 	IgnoreIngressWithoutClass bool
 }

diff --git a/pkg/common/ingress/controller/launch.go b/pkg/common/ingress/controller/launch.go
@@ -152,6 +152,9 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 		ingress controller should update the Ingress status IP/hostname when the controller
 		is being stopped. Default is true`)
 
+		backendShards = flags.Int("backend-shards", 0,
+			`Defines how much files should be used to configure the haproxy backends`)
+
 		sortBackends = flags.Bool("sort-backends", false,
 			`Defines if backends and it's endpoints should be sorted`)
 
@@ -311,6 +314,7 @@ func NewIngressController(backend ingress.Controller) *GenericController {
 		AllowCrossNamespace:       *allowCrossNamespace,
 		DisableNodeList:           *disableNodeList,
 		UpdateStatusOnShutdown:    *updateStatusOnShutdown,
+		BackendShards:             *backendShards,
 		SortBackends:              *sortBackends,
 		UseNodeInternalIP:         *useNodeInternalIP,
 		IgnoreIngressWithoutClass: *ignoreIngressWithoutClass,

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -120,7 +120,9 @@ func (hc *HAProxyController) configController() {
 	instanceOptions := haproxy.InstanceOptions{
 		HAProxyCmd:        "haproxy",
 		ReloadCmd:         "/haproxy-reload.sh",
-		HAProxyConfigFile: "/etc/haproxy/haproxy.cfg",
+		HAProxyCfgDir:     "/etc/haproxy",
+		HAProxyMapsDir:    "/etc/haproxy/maps",
+		BackendShards:     hc.cfg.BackendShards,
 		AcmeSigner:        acmeSigner,
 		AcmeQueue:         hc.acmeQueue,
 		LeaderElector:     hc.leaderelector,

diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go
@@ -45,9 +45,8 @@ type Config interface {
 
 type config struct {
 	// external state, non haproxy data
-	acmeData     *hatypes.AcmeData
-	mapsTemplate *template.Config
-	mapsDir      string
+	options  options
+	acmeData *hatypes.AcmeData
 	// haproxy internal state
 	globalOld   *hatypes.Global
 	global      *hatypes.Global
@@ -59,26 +58,24 @@ type config struct {
 }
 
 type options struct {
-	// reflect changes to config.Clear()
 	mapsTemplate *template.Config
 	mapsDir      string
+	shardCount   int
 }
 
 func createConfig(options options) *config {
-	mapsTemplate := options.mapsTemplate
-	if mapsTemplate == nil {
-		mapsTemplate = template.CreateConfig()
+	if options.mapsTemplate == nil {
+		options.mapsTemplate = template.CreateConfig()
 	}
 	return &config{
-		acmeData:     &hatypes.AcmeData{},
-		global:       &hatypes.Global{},
-		frontend:     &hatypes.Frontend{Name: "_front001"},
-		hosts:        hatypes.CreateHosts(),
-		backends:     hatypes.CreateBackends(),
-		tcpbackends:  hatypes.CreateTCPBackends(),
-		userlists:    hatypes.CreateUserlists(),
-		mapsTemplate: mapsTemplate,
-		mapsDir:      options.mapsDir,
+		options:     options,
+		acmeData:    &hatypes.AcmeData{},
+		global:      &hatypes.Global{},
+		frontend:    &hatypes.Frontend{Name: "_front001"},
+		hosts:       hatypes.CreateHosts(),
+		backends:    hatypes.CreateBackends(options.shardCount),
+		tcpbackends: hatypes.CreateTCPBackends(),
+		userlists:   hatypes.CreateUserlists(),
 	}
 }
 
@@ -143,23 +140,24 @@ func (c *config) SyncConfig() {
 // link to the frontend maps.
 func (c *config) WriteFrontendMaps() error {
 	mapBuilder := hatypes.CreateMaps()
+	mapsDir := c.options.mapsDir
 	fmaps := &hatypes.FrontendMaps{
-		HTTPFrontsMap:     mapBuilder.AddMap(c.mapsDir + "/_global_http_front.map"),
-		HTTPRootRedirMap:  mapBuilder.AddMap(c.mapsDir + "/_global_http_root_redir.map"),
-		HTTPSRedirMap:     mapBuilder.AddMap(c.mapsDir + "/_global_https_redir.map"),
-		SSLPassthroughMap: mapBuilder.AddMap(c.mapsDir + "/_global_sslpassthrough.map"),
-		VarNamespaceMap:   mapBuilder.AddMap(c.mapsDir + "/_global_k8s_ns.map"),
+		HTTPFrontsMap:     mapBuilder.AddMap(mapsDir + "/_global_http_front.map"),
+		HTTPRootRedirMap:  mapBuilder.AddMap(mapsDir + "/_global_http_root_redir.map"),
+		HTTPSRedirMap:     mapBuilder.AddMap(mapsDir + "/_global_https_redir.map"),
+		SSLPassthroughMap: mapBuilder.AddMap(mapsDir + "/_global_sslpassthrough.map"),
+		VarNamespaceMap:   mapBuilder.AddMap(mapsDir + "/_global_k8s_ns.map"),
 		//
-		HostBackendsMap:            mapBuilder.AddMap(c.mapsDir + "/_front001_host.map"),
-		RootRedirMap:               mapBuilder.AddMap(c.mapsDir + "/_front001_root_redir.map"),
-		SNIBackendsMap:             mapBuilder.AddMap(c.mapsDir + "/_front001_sni.map"),
-		TLSInvalidCrtErrorList:     mapBuilder.AddMap(c.mapsDir + "/_front001_inv_crt.list"),
-		TLSInvalidCrtErrorPagesMap: mapBuilder.AddMap(c.mapsDir + "/_front001_inv_crt_redir.map"),
-		TLSNoCrtErrorList:          mapBuilder.AddMap(c.mapsDir + "/_front001_no_crt.list"),
-		TLSNoCrtErrorPagesMap:      mapBuilder.AddMap(c.mapsDir + "/_front001_no_crt_redir.map"),
+		HostBackendsMap:            mapBuilder.AddMap(mapsDir + "/_front001_host.map"),
+		RootRedirMap:               mapBuilder.AddMap(mapsDir + "/_front001_root_redir.map"),
+		SNIBackendsMap:             mapBuilder.AddMap(mapsDir + "/_front001_sni.map"),
+		TLSInvalidCrtErrorList:     mapBuilder.AddMap(mapsDir + "/_front001_inv_crt.list"),
+		TLSInvalidCrtErrorPagesMap: mapBuilder.AddMap(mapsDir + "/_front001_inv_crt_redir.map"),
+		TLSNoCrtErrorList:          mapBuilder.AddMap(mapsDir + "/_front001_no_crt.list"),
+		TLSNoCrtErrorPagesMap:      mapBuilder.AddMap(mapsDir + "/_front001_no_crt_redir.map"),
 		//
-		CrtList:       mapBuilder.AddMap(c.mapsDir + "/_front001_bind_crt.list"),
-		UseServerList: mapBuilder.AddMap(c.mapsDir + "/_front001_use_server.list"),
+		CrtList:       mapBuilder.AddMap(mapsDir + "/_front001_bind_crt.list"),
+		UseServerList: mapBuilder.AddMap(mapsDir + "/_front001_use_server.list"),
 	}
 	fmaps.CrtList.AppendItem(c.frontend.DefaultCert)
 	// Some maps use yes/no answers instead of a list with found/missing keys
@@ -283,7 +281,7 @@ func (c *config) WriteFrontendMaps() error {
 			fmaps.CrtList.AppendItem(crtListEntry)
 		}
 	}
-	if err := writeMaps(mapBuilder, c.mapsTemplate); err != nil {
+	if err := writeMaps(mapBuilder, c.options.mapsTemplate); err != nil {
 		return err
 	}
 	c.frontend.Maps = fmaps
@@ -299,15 +297,15 @@ func (c *config) WriteBackendMaps() error {
 	mapBuilder := hatypes.CreateMaps()
 	for _, backend := range c.backends.Items() {
 		if backend.NeedACL() {
-			mapsPrefix := c.mapsDir + "/_back_" + backend.ID
+			mapsPrefix := c.options.mapsDir + "/_back_" + backend.ID
 			pathsMap := mapBuilder.AddMap(mapsPrefix + "_idpath.map")
 			for _, path := range backend.Paths {
 				pathsMap.AppendPath(path.Hostpath, path.ID)
 			}
 			backend.PathsMap = pathsMap
 		}
 	}
-	return writeMaps(mapBuilder, c.mapsTemplate)
+	return writeMaps(mapBuilder, c.options.mapsTemplate)
 }
 
 func writeMaps(maps *hatypes.HostsMaps, template *template.Config) error {
@@ -349,10 +347,7 @@ func (c *config) Userlists() *hatypes.Userlists {
 }
 
 func (c *config) Clear() {
-	config := createConfig(options{
-		mapsTemplate: c.mapsTemplate,
-		mapsDir:      c.mapsDir,
-	})
+	config := createConfig(c.options)
 	*c = *config
 }
 

diff --git a/pkg/haproxy/config_test.go b/pkg/haproxy/config_test.go
@@ -65,7 +65,7 @@ func TestClear(t *testing.T) {
 	})
 	c.Hosts().AcquireHost("app.local")
 	c.Backends().AcquireBackend("default", "app", "8080")
-	if c.mapsDir != "/tmp/maps" {
+	if c.options.mapsDir != "/tmp/maps" {
 		t.Error("expected mapsDir == /tmp/maps")
 	}
 	if len(c.Hosts().Items()) != 1 {
@@ -75,7 +75,7 @@ func TestClear(t *testing.T) {
 		t.Error("expected len(backends) == 1")
 	}
 	c.Clear()
-	if c.mapsDir != "/tmp/maps" {
+	if c.options.mapsDir != "/tmp/maps" {
 		t.Error("expected mapsDir == /tmp/maps")
 	}
 	if len(c.Hosts().Items()) != 0 {

diff --git a/pkg/haproxy/dynupdate_test.go b/pkg/haproxy/dynupdate_test.go
@@ -585,11 +585,10 @@ set server default_app_8080/srv002 weight 1`,
 	}
 	for i, test := range testCases {
 		c := setup(t)
-		instance := c.instance.(*instance)
 		if test.doconfig1 != nil {
 			test.doconfig1(c)
 		}
-		instance.config.Commit()
+		c.instance.config.Commit()
 		backendIDs := []types.BackendID{}
 		for _, backend := range c.config.Backends().Items() {
 			if backend != c.config.Backends().DefaultBackend() {
@@ -601,7 +600,7 @@ set server default_app_8080/srv002 weight 1`,
 			test.doconfig2(c)
 		}
 		var cmd string
-		dynUpdater := instance.newDynUpdater()
+		dynUpdater := c.instance.newDynUpdater()
 		dynUpdater.cmd = func(socket string, observer func(duration time.Duration), command ...string) ([]string, error) {
 			for _, c := range command {
 				cmd = cmd + c + "\n"