From 6fd5b98ad3c5c5c4d4d80357faa19d1398efda87 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Wed, 17 Mar 2021 15:34:31 -0300 Subject: [PATCH] add custom-proxy configuration Allows to configure any internal proxy - listen, frontend or backend - using a single configuration key as a multi section value. Each section of the configuration adds custom snippet to distinct proxies. --- docs/content/en/docs/configuration/keys.md | 25 +++-- pkg/converters/ingress/annotations/global.go | 17 +++ .../ingress/annotations/global_test.go | 58 ++++++++++ pkg/converters/ingress/types/global.go | 1 + pkg/haproxy/instance_test.go | 106 ++++++++++++++++++ pkg/haproxy/types/types.go | 1 + rootfs/etc/templates/haproxy/haproxy.tmpl | 52 ++++++++- 7 files changed, 248 insertions(+), 12 deletions(-) diff --git a/docs/content/en/docs/configuration/keys.md b/docs/content/en/docs/configuration/keys.md index 0f1726161..d57af761d 100644 --- a/docs/content/en/docs/configuration/keys.md +++ b/docs/content/en/docs/configuration/keys.md @@ -323,6 +323,7 @@ The table below describes all supported configuration keys. | [`config-defaults`](#configuration-snippet) | multiline HAProxy config for the defaults section | Global | | | [`config-frontend`](#configuration-snippet) | multiline HAProxy frontend config | Global | | | [`config-global`](#configuration-snippet) | multiline HAProxy global config | Global | | +| [`config-proxy`](#configuration-snippet) | multiline HAProxy custom proxy config | Global | | | [`config-sections`](#configuration-snippet) | multiline HAProxy section declarations | Global | | | [`cookie-key`](#affinity) | secret key | Global | `Ingress` | | [`cors-allow-credentials`](#cors) | [true\|false] | Path | | @@ -1016,11 +1017,19 @@ See also: | `config-defaults` | `Global` | | v0.8 | | `config-frontend` | `Global` | | | | `config-global` | `Global` | | | +| `config-proxy` | `Global` | | v0.13 | | `config-sections` | `Global` | | v0.13 | Add HAProxy configuration snippet to the configuration file. Use multiline content to add more than one line of configuration. +* `config-backend`: Adds a configuration snippet to a HAProxy backend section. +* `config-defaults`: Adds a configuration snippet to the end of the HAProxy defaults section. +* `config-frontend`: Adds a configuration snippet to the HTTP and HTTPS frontend sections. +* `config-global`: Adds a configuration snippet to the end of the HAProxy global section. +* `config-proxy`: Adds a configuration snippet to any HAProxy proxy - listen, frontend or backend. It accepts a multi section configuration, where the name of the section is the name of a HAProxy proxy without the listen/frontend/backend prefix. A section whose proxy is not found is ignored. The content of each section should be indented, the first line without indentation is the start of a new section which will configure another proxy. +* `config-sections`: Allows to declare new HAProxy sections. The configuration is used verbatim, without any indentation or validation. + Examples - ConfigMap: ```yaml @@ -1033,6 +1042,14 @@ Examples - ConfigMap: option redispatch ``` +```yaml + config-proxy: | + _tcp_default_postgresql_5432 + tcp-request content reject if !{ src 10.0.0.0/8 } + _front__tls + tcp-request content reject if !{ src 10.0.0.0/8 } { req.ssl_sni -m reg ^intra\..* } +``` + ```yaml config-sections: | cache icons @@ -1064,14 +1081,6 @@ Annotation: http-response cache-store icons if { var(txn.path) -m end .ico } ``` -The following keys add a configuration snippet to the ...: - -* `config-backend`: ... HAProxy backend section. -* `config-global`: ... end of the HAProxy global section. -* `config-defaults`: ... end of the HAProxy defaults section. -* `config-frontend`: ... HAProxy frontend sections. -* `config-sections`: ... HAProxy new section declarations. - --- ## Connection diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go index 3639b3702..298a629b5 100644 --- a/pkg/converters/ingress/annotations/global.go +++ b/pkg/converters/ingress/annotations/global.go @@ -345,4 +345,21 @@ func (c *updater) buildGlobalCustomConfig(d *globalData) { d.global.CustomDefaults = utils.LineToSlice(d.mapper.Get(ingtypes.GlobalConfigDefaults).Value) d.global.CustomFrontend = utils.LineToSlice(d.mapper.Get(ingtypes.GlobalConfigFrontend).Value) d.global.CustomSections = utils.LineToSlice(d.mapper.Get(ingtypes.GlobalConfigSections).Value) + proxy := map[string][]string{} + var curSection string + for _, line := range utils.LineToSlice(d.mapper.Get(ingtypes.GlobalConfigProxy).Value) { + if line == "" { + continue + } + if line[0] != ' ' && line[0] != '\t' { + curSection = line + continue + } + proxy[curSection] = append(proxy[curSection], strings.TrimSpace(line)) + } + if lines, hasEmpty := proxy[""]; hasEmpty { + c.logger.Warn("non scoped %d line(s) in the config-proxy configuration were ignored", len(lines)) + delete(proxy, "") + } + d.global.CustomProxy = proxy } diff --git a/pkg/converters/ingress/annotations/global_test.go b/pkg/converters/ingress/annotations/global_test.go index cae8aac6b..e27ab3943 100644 --- a/pkg/converters/ingress/annotations/global_test.go +++ b/pkg/converters/ingress/annotations/global_test.go @@ -92,6 +92,64 @@ func TestBind(t *testing.T) { } } +func TestCustomConfigProxy(t *testing.T) { + testCases := []struct { + config string + expected map[string][]string + logging string + }{ + // 0 + {}, + // 1 + { + config: ` +proxy_1 + acl test`, + expected: map[string][]string{ + "proxy_1": {"acl test"}, + }, + }, + // 2 + { + config: ` +backend_1 + acl ok always_true + http-request deny if !ok +backend_2 + ## two spaces + ## four spaces + ## two tabs`, + expected: map[string][]string{ + "backend_1": {"acl ok always_true", "http-request deny if !ok"}, + "backend_2": {"## two spaces", "## four spaces", "## two tabs"}, + }, + }, + // 3 + { + config: ` + ## trailing line 1 +proxy_1 + acl ok always_true +`, + expected: map[string][]string{ + "proxy_1": {"acl ok always_true"}, + }, + logging: `WARN non scoped 1 line(s) in the config-proxy configuration were ignored`, + }, + } + for i, test := range testCases { + c := setup(t) + d := c.createGlobalData(map[string]string{ingtypes.GlobalConfigProxy: test.config}) + c.createUpdater().buildGlobalCustomConfig(d) + if test.expected == nil { + test.expected = map[string][]string{} + } + c.compareObjects("custom config", i, d.global.CustomProxy, test.expected) + c.logger.CompareLogging(test.logging) + c.teardown() + } +} + func TestModSecurity(t *testing.T) { testCases := []struct { endpoints string diff --git a/pkg/converters/ingress/types/global.go b/pkg/converters/ingress/types/global.go index 81989e3be..18fde30bf 100644 --- a/pkg/converters/ingress/types/global.go +++ b/pkg/converters/ingress/types/global.go @@ -34,6 +34,7 @@ const ( GlobalConfigDefaults = "config-defaults" GlobalConfigFrontend = "config-frontend" GlobalConfigGlobal = "config-global" + GlobalConfigProxy = "config-proxy" GlobalConfigSections = "config-sections" GlobalCookieKey = "cookie-key" GlobalCPUMap = "cpu-map" diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index 30d4b5d89..bba1412c7 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -2309,6 +2309,112 @@ backend d1_app_8080 c.logger.CompareLogging(defaultLogging) } +func TestInstanceCustomProxy(t *testing.T) { + c := setup(t) + defer c.teardown() + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.Backends().AcquireBackend("d1", "app", "8080") + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h = c.config.Hosts().AcquireHost("d1.local") + h.AddPath(b, "/", hatypes.MatchBegin) + h.SetSSLPassthrough(true) + + tcp := c.config.tcpbackends.Acquire("default_pgsql", 5432) + tcp.AddEndpoint("172.17.0.21", 5432) + + c.config.Global().CustomProxy = map[string][]string{ + "missing": {"## comment"}, + "_tcp_default_pgsql_5432": {"## custom for _tcp_default_pgsql_5432"}, + "d1_app_8080": {"## custom for d1_app_8080"}, + "_redirect_https": {"## custom for _redirect_https"}, + "_error404": {"## custom for _error404", "## line 2"}, + "_front__tls": {"## custom for _front__tls"}, + "_front_http": {"## custom for _front_http"}, + "_front_https": {"## custom for _front_https"}, + "stats": {"## custom for stats"}, + "healthz": {"## custom for healthz"}, + } + + c.Update() + c.checkConfig(` +<> +<> +listen _tcp_default_pgsql_5432 + bind :5432 + mode tcp + ## custom for _tcp_default_pgsql_5432 + server srv001 172.17.0.21:5432 +backend d1_app_8080 + mode http + ## custom for d1_app_8080 + server s1 172.17.0.11:8080 weight 100 +backend _redirect_https + mode http + ## custom for _redirect_https + http-request redirect scheme https +backend _error404 + mode http + ## custom for _error404 + ## line 2 + http-request use-service lua.send-404 +listen _front__tls + mode tcp + bind :443 + tcp-request inspect-delay 5s + tcp-request content set-var(req.sslpassback) req.ssl_sni,lower,map_str(/etc/haproxy/maps/_front_sslpassthrough__exact.map) + ## custom for _front__tls + tcp-request content accept if { req.ssl_hello_type 1 } + use_backend %[var(req.sslpassback)] if { var(req.sslpassback) -m found } + server _default_server_https_socket unix@/var/run/haproxy/_https_socket.sock send-proxy-v2 +frontend _front_http + mode http + bind :80 + http-request set-var(req.path) path + http-request set-var(req.host) hdr(host),field(1,:),lower + http-request set-var(req.base) var(req.host),concat(,req.path) + http-request set-header X-Forwarded-Proto http + http-request del-header X-SSL-Client-CN + http-request del-header X-SSL-Client-DN + http-request del-header X-SSL-Client-SHA1 + http-request del-header X-SSL-Client-Cert + http-request set-var(req.backend) var(req.base),lower,map_beg(/etc/haproxy/maps/_front_http_host__begin.map) + ## custom for _front_http + use_backend %[var(req.backend)] if { var(req.backend) -m found } + default_backend _error404 +frontend _front_https + mode http + bind unix@/var/run/haproxy/_https_socket.sock accept-proxy ssl alpn h2,http/1.1 crt-list /etc/haproxy/maps/_front_bind_crt.list ca-ignore-err all crt-ignore-err all + http-request set-header X-Forwarded-Proto https + http-request del-header X-SSL-Client-CN + http-request del-header X-SSL-Client-DN + http-request del-header X-SSL-Client-SHA1 + http-request del-header X-SSL-Client-Cert + ## custom for _front_https + use_backend %[var(req.hostbackend)] if { var(req.hostbackend) -m found } + default_backend _error404 +listen stats + mode http + bind :1936 + stats enable + stats uri / + no log + option httpclose + stats show-legends + ## custom for stats +frontend healthz + mode http + bind :10253 + monitor-uri /healthz + http-request use-service lua.send-404 + no log + ## custom for healthz +`) + c.logger.CompareLogging(defaultLogging) +} + func TestInstanceSSLPassthrough(t *testing.T) { c := setup(t) defer c.teardown() diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 5e75c5b2c..591c2d60a 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -78,6 +78,7 @@ type Global struct { CustomConfig []string CustomDefaults []string CustomFrontend []string + CustomProxy map[string][]string CustomSections []string } diff --git a/rootfs/etc/templates/haproxy/haproxy.tmpl b/rootfs/etc/templates/haproxy/haproxy.tmpl index 091eb3928..3ae902d61 100644 --- a/rootfs/etc/templates/haproxy/haproxy.tmpl +++ b/rootfs/etc/templates/haproxy/haproxy.tmpl @@ -245,7 +245,8 @@ userlist {{ $userlist.Name }} # {{- range $backend := $tcpbackends }} -listen _tcp_{{ $backend.Name }}_{{ $backend.Port }} +{{- $proxy_name := printf "_tcp_%s_%d" $backend.Name $backend.Port }} +listen {{ $proxy_name }} {{- $ssl := $backend.SSL }} bind {{ $global.Bind.TCPBindIP }}:{{ $backend.Port }} {{- if $ssl.Filename }} ssl crt {{ $ssl.Filename }} @@ -267,6 +268,11 @@ listen _tcp_{{ $backend.Name }}_{{ $backend.Port }} {{- end }} {{- end }} +{{- /*------------------------------------*/}} +{{- range $snippet := index $global.CustomProxy $proxy_name }} + {{ $snippet }} +{{- end }} + {{- /*------------------------------------*/}} {{- $outProxyProtVersion := $backend.ProxyProt.EncodeVersion }} {{- range $ep := $backend.Endpoints }} @@ -644,6 +650,9 @@ backend {{ $backend.ID }} {{- range $snippet := $backend.CustomConfig }} {{ $snippet }} {{- end }} +{{- range $snippet := index $global.CustomProxy $backend.ID }} + {{ $snippet }} +{{- end }} {{- /*------------------------------------*/}} {{- $rewriteCfg := $backend.PathConfig "RewriteURL" }} @@ -793,6 +802,9 @@ backend {{ $backend.ID }} # backend _redirect_https mode http +{{- range $snippet := index $global.CustomProxy "_redirect_https" }} + {{ $snippet }} +{{- end }} http-request redirect scheme https {{- if $global.SSL.RedirectCode }} code {{ $global.SSL.RedirectCode }}{{ end }} {{- end }} @@ -805,6 +817,9 @@ backend _redirect_https # backend _acme_challenge mode http +{{- range $snippet := index $global.CustomProxy "_acme_challenge" }} + {{ $snippet }} +{{- end }} server _acme_server unix@{{ $global.Acme.Socket }} {{- end }} @@ -816,6 +831,9 @@ backend _acme_challenge # backend _error404 mode http +{{- range $snippet := index $global.CustomProxy "_error404" }} + {{ $snippet }} +{{- end }} {{- if $global.DefaultBackendRedir }} redirect location {{ $global.DefaultBackendRedir }} code {{ $global.DefaultBackendRedirCode }} {{- else }} @@ -846,7 +864,8 @@ backend _error404 # # # TCP/TLS frontend # -listen _front__tls +{{- $proxy__front__tls := "_front__tls" }} +listen {{ $proxy__front__tls }} mode tcp bind {{ $global.Bind.HTTPSBind }}{{ if $global.Bind.AcceptProxy }} accept-proxy{{ end }} @@ -871,6 +890,11 @@ listen _front__tls {{- if not $match.First }} if !{ var(req.sslpassback) -m found }{{ end }} {{- end }} +{{- /*------------------------------------*/}} +{{- range $snippet := index $global.CustomProxy $proxy__front__tls }} + {{ $snippet }} +{{- end }} + {{- /*------------------------------------*/}} tcp-request content accept if { req.ssl_hello_type 1 } @@ -887,7 +911,8 @@ listen _front__tls # # # HTTP{{ if $hasFrontingProxy }} & Fronting Proxy{{ end }} frontend # -frontend _front_http +{{- $proxy__front_http := "_front_http" }} +frontend {{ $proxy__front_http }} mode http {{- $hasPlainHTTPSocket := not $global.Bind.ShareHTTPPort }} {{- if and $global.Bind.HTTPBind $hasPlainHTTPSocket }} @@ -978,6 +1003,9 @@ frontend _front_http {{- range $snippet := $global.CustomFrontend }} {{ $snippet }} {{- end }} +{{- range $snippet := index $global.CustomProxy $proxy__front_http }} + {{ $snippet }} +{{- end }} {{- /*------------------------------------*/}} {{- if $acmeexclusive }} @@ -994,7 +1022,8 @@ frontend _front_http # # # HTTPS frontend # -frontend _front_https +{{- $proxy__front_https := "_front_https" }} +frontend {{ $proxy__front_https }} mode http {{- /*------------------------------------*/}} @@ -1132,6 +1161,9 @@ frontend _front_https {{- range $snippet := $global.CustomFrontend }} {{ $snippet }} {{- end }} +{{- range $snippet := index $global.CustomProxy $proxy__front_https }} + {{ $snippet }} +{{- end }} {{- /*------------------------------------*/}} {{- if $fmaps.TLSAuthList.HasHost }} @@ -1208,6 +1240,9 @@ listen stats no log option httpclose stats show-legends +{{- range $snippet := index $global.CustomProxy "stats" }} + {{ $snippet }} +{{- end }} {{- if $global.Prometheus.Port }} @@ -1222,6 +1257,9 @@ frontend prometheus http-request use-service lua.send-prometheus-root if { path / } http-request use-service lua.send-404 no log +{{- range $snippet := index $global.CustomProxy "prometheus" }} + {{ $snippet }} +{{- end }} {{- end }} # # # # # # # # # # # # # # # # # # # @@ -1234,6 +1272,9 @@ frontend healthz monitor-uri /healthz http-request use-service lua.send-404 no log +{{- range $snippet := index $global.CustomProxy "healthz" }} + {{ $snippet }} +{{- end }} {{- if $global.ModSecurity.Endpoints }} @@ -1245,6 +1286,9 @@ backend spoe-modsecurity mode tcp timeout connect {{ $global.ModSecurity.Timeout.Connect }} timeout server {{ $global.ModSecurity.Timeout.Server }} +{{- range $snippet := index $global.CustomProxy "spoe-modsecurity" }} + {{ $snippet }} +{{- end }} {{- range $i, $endpoint := $global.ModSecurity.Endpoints }} server modsec-spoa{{ $i }} {{ $endpoint }} {{- end }}