From abebe87612b158f0c0c6b58d671b3c7db2af61f7 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 10 May 2020 16:07:52 -0300 Subject: [PATCH] add use-forwarded-proto config key `use-forwarded-proto` extends `fronting-proxy-port` allowing a https-only requests behavior, without analysing `X-Forwarded-Proto`, which is forwarded to the backend. --- docs/content/en/docs/configuration/keys.md | 21 ++- pkg/converters/ingress/annotations/global.go | 1 + pkg/converters/ingress/defaults.go | 1 + pkg/converters/ingress/types/global.go | 1 + pkg/haproxy/instance_test.go | 187 ++++++++++++++++++- pkg/haproxy/types/types.go | 13 +- rootfs/etc/haproxy/template/haproxy.tmpl | 20 +- 7 files changed, 217 insertions(+), 27 deletions(-) diff --git a/docs/content/en/docs/configuration/keys.md b/docs/content/en/docs/configuration/keys.md index 40bec6fa0..1e5b20663 100644 --- a/docs/content/en/docs/configuration/keys.md +++ b/docs/content/en/docs/configuration/keys.md @@ -235,6 +235,7 @@ The table below describes all supported configuration keys. | [`timeout-tunnel`](#timeout) | time with suffix | Backend | `1h` | | [`tls-alpn`](#tls-alpn) | TLS ALPN advertisement | Global | `h2,http/1.1` | | [`use-chroot`](#security) | [true\|false] | Global | `false` | +| [`use-forwarded-proto`](#fronting-proxy-port) | [true\|false] | Global | `true` | | [`use-haproxy-user`](#security) | [true\|false] | Global | `false` | | [`use-htx`](#use-htx) | [true\|false] | Global | `false` | | [`use-proxy-protocol`](#proxy-protocol) | [true\|false] | Global | `false` | @@ -925,10 +926,11 @@ See also: ## Fronting proxy port -| Configuration key | Scope | Default | Since | -|-----------------------|----------|---------|--------| -| `fronting-proxy-port` | `Global` | | `v0.8` | -| `https-to-http-port` | `Global` | | | +| Configuration key | Scope | Default | Since | +|-----------------------|----------|---------|---------| +| `fronting-proxy-port` | `Global` | | `v0.8` | +| `https-to-http-port` | `Global` | | | +| `use-forwarded-proto` | `Global` | `true` | `v0.10` | A port number to listen to http requests from a fronting proxy that does the ssl offload, eg haproxy ingress behind a cloud load balancers that manages the TLS @@ -937,8 +939,15 @@ certificates. `https-to-http-port` is an alias to `fronting-proxy-port`. `fronting-proxy-port` and [`http-port`](#bind-port) can share the same port number, see below what changes in the behaviour. -Requests made to `fronting-proxy-port` port number evaluate the `X-Forwarded-Proto` -header to decide how to handle the request: +`use-forwarded-proto` defines if haproxy should use `X-Forwarded-Proto` header to decide +how to handle requests made to `fronting-proxy-port` port number. + +If `use-forwarded-proto` is `false`, the request takes the `https` route and is handled as if +`X-Forwarded-Proto` header is `https`, see below. The actual header content is ignored by +haproxy and forwarded to the backend if provided. + +If `use-forwarded-proto` is `true`, the default value, requests made to `fronting-proxy-port` +port number evaluate the `X-Forwarded-Proto` header to decide how to handle the request: * If `X-Forwarded-Proto` header is `https`: * HAProxy will handle the request just like the ssl-offload was made by HAProxy itself - HSTS header is provided if configured and diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go index f63799f65..d36e779d8 100644 --- a/pkg/converters/ingress/annotations/global.go +++ b/pkg/converters/ingress/annotations/global.go @@ -207,6 +207,7 @@ func (c *updater) buildGlobalHTTPStoHTTP(d *globalData) { } // TODO Change all `ToHTTP` naming to `FrontingProxy` d.global.Bind.FrontingBind = bind + d.global.Bind.FrontingUseProto = d.mapper.Get(ingtypes.GlobalUseForwardedProto).Bool() // Socket ID should be a high number to avoid colision // between the same socket ID from distinct frontends // TODO match socket and frontend ID in the backend diff --git a/pkg/converters/ingress/defaults.go b/pkg/converters/ingress/defaults.go index 2432ace2e..d65e720cd 100644 --- a/pkg/converters/ingress/defaults.go +++ b/pkg/converters/ingress/defaults.go @@ -95,6 +95,7 @@ func createDefaults() map[string]string { types.GlobalTimeoutClientFin: "50s", types.GlobalTimeoutStop: "10m", types.GlobalTLSALPN: "h2,http/1.1", + types.GlobalUseForwardedProto: "true", types.GlobalUseHTX: "true", } } diff --git a/pkg/converters/ingress/types/global.go b/pkg/converters/ingress/types/global.go index 27cccd2e6..35217630c 100644 --- a/pkg/converters/ingress/types/global.go +++ b/pkg/converters/ingress/types/global.go @@ -88,6 +88,7 @@ const ( GlobalTimeoutStop = "timeout-stop" GlobalTLSALPN = "tls-alpn" GlobalUseChroot = "use-chroot" + GlobalUseForwardedProto = "use-forwarded-proto" GlobalUseHAProxyUser = "use-haproxy-user" GlobalUseHTX = "use-htx" GlobalUseProxyProtocol = "use-proxy-protocol" diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index e28807a60..d88fbe5ad 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -741,9 +741,9 @@ empty/ default_empty_8080`) c.logger.CompareLogging(defaultLogging) } -func TestInstanceToHTTPSocket(t *testing.T) { +func TestInstanceFrontingProxyUseProto(t *testing.T) { testCases := []struct { - toHTTPBind string + frontingBind string domain string expectedFront string expectedMap string @@ -753,8 +753,8 @@ func TestInstanceToHTTPSocket(t *testing.T) { }{ // 0 { - toHTTPBind: ":8000", - domain: "d1.local", + frontingBind: ":8000", + domain: "d1.local", expectedFront: ` mode http bind :80 @@ -788,8 +788,8 @@ func TestInstanceToHTTPSocket(t *testing.T) { }, // 1 { - toHTTPBind: ":8000", - domain: "*.d1.local", + frontingBind: ":8000", + domain: "*.d1.local", expectedFront: ` mode http bind :80 @@ -831,8 +831,8 @@ func TestInstanceToHTTPSocket(t *testing.T) { }, // 2 { - toHTTPBind: ":80", - domain: "d1.local", + frontingBind: ":80", + domain: "d1.local", expectedFront: ` mode http bind :80 @@ -887,8 +887,9 @@ func TestInstanceToHTTPSocket(t *testing.T) { } h.TLS.CAHash = "1" h.TLS.CAFilename = "/var/haproxy/ssl/ca.pem" - c.config.Global().Bind.FrontingBind = test.toHTTPBind + c.config.Global().Bind.FrontingBind = test.frontingBind c.config.Global().Bind.FrontingSockID = 11 + c.config.Global().Bind.FrontingUseProto = true c.Update() c.checkConfig(` @@ -937,6 +938,174 @@ frontend _front001 } } +func TestInstanceFrontingProxyIgnoreProto(t *testing.T) { + testCases := []struct { + frontingBind string + domain string + expectedFront string + expectedMap string + expectedRegexMap string + expectedACL string + expectedSetvar string + }{ + // 0 + { + frontingBind: ":8000", + domain: "d1.local", + expectedFront: ` + mode http + bind :80 + bind :8000 id 11 + http-request set-var(req.base) base,lower,regsub(:[0-9]+/,/) + http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/_global_http_front.map) + use_backend %[var(req.backend)] if { var(req.backend) -m found }`, + expectedMap: "d1.local/ d1_app_8080", + expectedACL: ` + acl tls-has-crt ssl_c_used + acl tls-need-crt ssl_fc_sni -i -f /etc/haproxy/maps/_front001_no_crt.list + acl tls-host-need-crt var(req.host) -i -f /etc/haproxy/maps/_front001_no_crt.list + acl tls-has-invalid-crt ssl_c_ca_err gt 0 + acl tls-has-invalid-crt ssl_c_err gt 0 + acl tls-check-crt ssl_fc_sni -i -f /etc/haproxy/maps/_front001_inv_crt.list`, + expectedSetvar: ` + http-request set-var(req.path) path + http-request set-var(req.snibase) ssl_fc_sni,concat(,req.path),lower + http-request set-var(req.snibackend) var(req.snibase),map_beg(/etc/haproxy/maps/_front001_sni.map) + http-request set-var(req.snibackend) var(req.base),map_beg(/etc/haproxy/maps/_front001_sni.map) if !{ var(req.snibackend) -m found } !tls-has-crt !tls-host-need-crt + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,lower,map(/etc/haproxy/maps/_front001_no_crt_redir.map,_internal) if !tls-has-crt tls-need-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,lower,map(/etc/haproxy/maps/_front001_inv_crt_redir.map,_internal) if tls-has-invalid-crt tls-check-crt`, + }, + // 1 + { + frontingBind: ":8000", + domain: "*.d1.local", + expectedFront: ` + mode http + bind :80 + bind :8000 id 11 + http-request set-var(req.base) base,lower,regsub(:[0-9]+/,/) + http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/_global_http_front.map) + http-request set-var(req.backend) var(req.base),map_reg(/etc/haproxy/maps/_global_http_front_regex.map) if !{ var(req.backend) -m found } + use_backend %[var(req.backend)] if { var(req.backend) -m found }`, + expectedRegexMap: `^[^.]+\.d1\.local/ d1_app_8080`, + expectedACL: ` + acl tls-has-crt ssl_c_used + acl tls-need-crt ssl_fc_sni -i -f /etc/haproxy/maps/_front001_no_crt.list + acl tls-need-crt ssl_fc_sni -i -m reg -f /etc/haproxy/maps/_front001_no_crt_regex.list + acl tls-host-need-crt var(req.host) -i -f /etc/haproxy/maps/_front001_no_crt.list + acl tls-host-need-crt var(req.host) -i -m reg -f /etc/haproxy/maps/_front001_no_crt_regex.list + acl tls-has-invalid-crt ssl_c_ca_err gt 0 + acl tls-has-invalid-crt ssl_c_err gt 0 + acl tls-check-crt ssl_fc_sni -i -f /etc/haproxy/maps/_front001_inv_crt.list + acl tls-check-crt ssl_fc_sni -i -m reg -f /etc/haproxy/maps/_front001_inv_crt_regex.list`, + expectedSetvar: ` + http-request set-var(req.path) path + http-request set-var(req.snibase) ssl_fc_sni,concat(,req.path),lower + http-request set-var(req.snibackend) var(req.snibase),map_beg(/etc/haproxy/maps/_front001_sni.map) + http-request set-var(req.snibackend) var(req.snibase),map_reg(/etc/haproxy/maps/_front001_sni_regex.map) if !{ var(req.snibackend) -m found } + http-request set-var(req.snibackend) var(req.base),map_beg(/etc/haproxy/maps/_front001_sni.map) if !{ var(req.snibackend) -m found } !tls-has-crt !tls-host-need-crt + http-request set-var(req.snibackend) var(req.base),map_reg(/etc/haproxy/maps/_front001_sni_regex.map) if !{ var(req.snibackend) -m found } !tls-has-crt !tls-host-need-crt + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,lower,map(/etc/haproxy/maps/_front001_no_crt_redir.map,_internal) if !tls-has-crt tls-need-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,lower,map(/etc/haproxy/maps/_front001_inv_crt_redir.map,_internal) if tls-has-invalid-crt tls-check-crt`, + }, + // 2 + { + frontingBind: ":80", + domain: "d1.local", + expectedFront: ` + mode http + bind :80 + http-request set-var(req.base) base,lower,regsub(:[0-9]+/,/) + http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/_global_http_front.map) + use_backend %[var(req.backend)] if { var(req.backend) -m found }`, + expectedMap: "d1.local/ d1_app_8080", + expectedACL: ` + acl tls-has-crt ssl_c_used + acl tls-need-crt ssl_fc_sni -i -f /etc/haproxy/maps/_front001_no_crt.list + acl tls-host-need-crt var(req.host) -i -f /etc/haproxy/maps/_front001_no_crt.list + acl tls-has-invalid-crt ssl_c_ca_err gt 0 + acl tls-has-invalid-crt ssl_c_err gt 0 + acl tls-check-crt ssl_fc_sni -i -f /etc/haproxy/maps/_front001_inv_crt.list`, + expectedSetvar: ` + http-request set-var(req.path) path + http-request set-var(req.snibase) ssl_fc_sni,concat(,req.path),lower + http-request set-var(req.snibackend) var(req.snibase),map_beg(/etc/haproxy/maps/_front001_sni.map) + http-request set-var(req.snibackend) var(req.base),map_beg(/etc/haproxy/maps/_front001_sni.map) if !{ var(req.snibackend) -m found } !tls-has-crt !tls-host-need-crt + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,lower,map(/etc/haproxy/maps/_front001_no_crt_redir.map,_internal) if !tls-has-crt tls-need-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,lower,map(/etc/haproxy/maps/_front001_inv_crt_redir.map,_internal) if tls-has-invalid-crt tls-check-crt`, + }, + } + for _, test := range testCases { + c := setup(t) + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.Backends().AcquireBackend("d1", "app", "8080") + h = c.config.Hosts().AcquireHost(test.domain) + h.AddPath(b, "/") + b.Endpoints = []*hatypes.Endpoint{endpointS1} + b.HSTS = []*hatypes.BackendConfigHSTS{ + { + Paths: hatypes.NewBackendPaths(b.FindHostPath(test.domain + "/")), + Config: hatypes.HSTS{ + Enabled: true, + MaxAge: 15768000, + Subdomains: true, + Preload: true, + }, + }, + } + h.TLS.CAHash = "1" + h.TLS.CAFilename = "/var/haproxy/ssl/ca.pem" + c.config.Global().Bind.FrontingBind = test.frontingBind + c.config.Global().Bind.FrontingSockID = 11 + c.config.Global().Bind.FrontingUseProto = false + + c.Update() + c.checkConfig(` +<> +<> +backend d1_app_8080 + mode http + acl local-offload ssl_fc + http-request set-header X-SSL-Client-CN %{+Q}[ssl_c_s_dn(cn)] if local-offload + http-request set-header X-SSL-Client-DN %{+Q}[ssl_c_s_dn] if local-offload + http-request set-header X-SSL-Client-SHA1 %{+Q}[ssl_c_sha1,hex] if local-offload + http-response set-header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + server s1 172.17.0.11:8080 weight 100 +<> +frontend _front_http` + test.expectedFront + ` + default_backend _error404 +frontend _front001 + mode http + bind :443 ssl alpn h2,http/1.1 crt-list /etc/haproxy/maps/_front001_bind_crt.list ca-ignore-err all crt-ignore-err all + http-request set-var(req.base) base,lower,regsub(:[0-9]+/,/) + http-request set-var(req.hostbackend) var(req.base),map_beg(/etc/haproxy/maps/_front001_host.map) + http-request set-var(req.host) hdr(host),lower,regsub(:[0-9]+$,) + http-request set-header X-Forwarded-Proto https + http-request del-header X-SSL-Client-CN + http-request del-header X-SSL-Client-DN + http-request del-header X-SSL-Client-SHA1 + http-request del-header X-SSL-Client-Cert` + test.expectedACL + test.expectedSetvar + ` + http-request use-service lua.send-421 if tls-has-crt { ssl_fc_has_sni } !{ ssl_fc_sni,strcmp(req.host) eq 0 } + http-request use-service lua.send-496 if { var(req.tls_nocrt_redir) _internal } + http-request use-service lua.send-421 if !tls-has-crt tls-host-need-crt + http-request use-service lua.send-495 if { var(req.tls_invalidcrt_redir) _internal } + use_backend %[var(req.hostbackend)] if { var(req.hostbackend) -m found } + use_backend %[var(req.snibackend)] if { var(req.snibackend) -m found } + default_backend _error404 +<> +`) + c.checkMap("_global_http_front.map", test.expectedMap) + if test.expectedRegexMap != "" { + c.checkMap("_global_http_front_regex.map", test.expectedRegexMap) + } + c.logger.CompareLogging(defaultLogging) + c.teardown() + } +} + func TestInstanceTCPBackend(t *testing.T) { testCases := []struct { doconfig func(c *testConfig) diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 10c89f508..74cc29387 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -66,12 +66,13 @@ type Global struct { // GlobalBindConfig ... type GlobalBindConfig struct { - AcceptProxy bool - HTTPBind string - HTTPSBind string - TCPBindIP string - FrontingBind string - FrontingSockID int + AcceptProxy bool + HTTPBind string + HTTPSBind string + TCPBindIP string + FrontingBind string + FrontingSockID int + FrontingUseProto bool } // ProcsConfig ... diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index ad0b739ae..30160c3c0 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -288,9 +288,11 @@ backend {{ $backend.ID }} {{- /*------------------------------------*/}} {{- $hasFrontingProxy := $global.Bind.HasFrontingProxy }} -{{- if $backend.HSTS }} +{{- $frontingUseProto := and $hasFrontingProxy $global.Bind.FrontingUseProto }} +{{- $frontingIgnoreProto := and $hasFrontingProxy (not $global.Bind.FrontingUseProto) }} +{{- if and $backend.HSTS (not $frontingIgnoreProto) }} acl https-request ssl_fc -{{- if $hasFrontingProxy }} +{{- if $frontingUseProto }} acl https-request var(txn.proto) https {{- end }} {{- end }} @@ -299,7 +301,7 @@ backend {{ $backend.ID }} {{- end }} {{- /*------------------------------------*/}} -{{- if and $hasFrontingProxy $backend.HSTS }} +{{- if and $frontingUseProto $backend.HSTS }} http-request set-var(txn.proto) hdr(X-Forwarded-Proto) {{- end }} @@ -470,7 +472,7 @@ backend {{ $backend.ID }} {{- $hsts := $hstsCfg.Config }} {{- if $hsts.Enabled }} {{- $paths := $hstsCfg.Paths }} -{{- $needSSLACL := not ($backend.HasSSLRedirectPaths $paths) }} +{{- $needSSLACL := and (not $frontingIgnoreProto) (not ($backend.HasSSLRedirectPaths $paths)) }} http-response set-header Strict-Transport-Security "max-age={{ $hsts.MaxAge }} {{- if $hsts.Subdomains }}; includeSubDomains{{ end }} {{- if $hsts.Preload }}; preload{{ end }}" @@ -675,6 +677,8 @@ listen _front__tls {{- end }} {{- $hasFrontingProxy := $global.Bind.HasFrontingProxy }} +{{- $frontingUseProto := and $hasFrontingProxy $global.Bind.FrontingUseProto }} +{{- $frontingIgnoreProto := and $hasFrontingProxy (not $global.Bind.FrontingUseProto) }} # # # # # # # # # # # # # # # # # # # # # @@ -693,7 +697,7 @@ frontend _front_http {{- end }} {{- /*------------------------------------*/}} -{{- if $hasFrontingProxy }} +{{- if $frontingUseProto }} {{- if $hasPlainHTTPSocket }} acl fronting-proxy so_id {{ $global.Bind.FrontingSockID }} {{- else }} @@ -711,7 +715,7 @@ frontend _front_http {{- end }} {{- /*------------------------------------*/}} -{{- if $hasFrontingProxy }} +{{- if $frontingUseProto }} http-request redirect scheme https {{- if $global.SSL.RedirectCode }} code {{ $global.SSL.RedirectCode }}{{ end }} {{- "" }} if fronting-proxy !{ hdr(X-Forwarded-Proto) https } @@ -727,6 +731,7 @@ frontend _front_http {{- /*------------------------------------*/}} {{- $acmeexclusive := and $cfg.Acme.Enabled (not $cfg.Acme.Shared) }} +{{- if not $frontingIgnoreProto }} {{- if $fmaps.HTTPSRedirMap.HasRegex }} http-request set-var(req.redir) {{- "" }} var(req.base),map_beg({{ $fmaps.HTTPSRedirMap.MatchFile }}) @@ -749,6 +754,7 @@ frontend _front_http {{- if $hasFrontingProxy }} !fronting-proxy{{ end }} {{- "" }} { var(req.base),map_beg({{ $fmaps.HTTPSRedirMap.MatchFile }}) yes } {{- end }} +{{- end }} {{- /*------------------------------------*/}} {{- if $fmaps.HTTPRootRedirMap.HasHost }} @@ -776,6 +782,7 @@ frontend _front_http {{- end }} {{- /*------------------------------------*/}} +{{- if not $frontingIgnoreProto }} http-request set-header X-Forwarded-Proto http {{- if $hasFrontingProxy }} if !fronting-proxy{{ end }} http-request del-header {{ $global.SSL.HeadersPrefix }}-Client-CN @@ -786,6 +793,7 @@ frontend _front_http {{- if $hasFrontingProxy }} if !fronting-proxy{{ end }} http-request del-header {{ $global.SSL.HeadersPrefix }}-Client-Cert {{- if $hasFrontingProxy }} if !fronting-proxy{{ end }} +{{- end }} {{- /*------------------------------------*/}} http-request set-var(req.backend) var(req.base),map_beg({{ $fmaps.HTTPFrontsMap.MatchFile }})