From 33bbc0256ca8489c8de03d2f5fbe8164b74c3bf3 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sat, 21 Jul 2018 11:37:06 -0300 Subject: [PATCH] Add SSL/TLS config on tcp services --- README.md | 17 +++++++----- pkg/common/ingress/controller/controller.go | 30 +++++++++++++++++++-- pkg/common/ingress/types.go | 1 + pkg/common/ingress/types_equals.go | 3 +++ rootfs/etc/haproxy/template/haproxy.tmpl | 6 ++++- 5 files changed, 47 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 828253827..fa9738618 100644 --- a/README.md +++ b/README.md @@ -711,12 +711,15 @@ Configure `--tcp-services-configmap` argument with `namespace/configmapname` res services and ports that HAProxy should listen to. Use the HAProxy's port number as the key of the configmap. -The value of the configmap entry has the following syntax: `/:[:[][:]]`, where: +The value of the configmap entry is a colon separated list of the following items: -* `/` is the well known notation of the service that will receive incomming connections. -* `` is the port number the upstream service is listening - this is not related to the listening port of HAProxy. -* `` should be defined as `PROXY` if HAProxy should expect requests using the [PROXY](http://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) protocol. This is usually true only if there is another load balancer in front of HAProxy which supports the PROXY protocol. PROXY protocol v1 and v2 are supported. -* `` should be defined as `PROXY` or `PROXY-V2` if the upstream service expect connections using the PROXY protocol v2. Use `PROXY-V1` instead if the upstream service only support v1 protocol. +1. `/`, mandatory, is the well known notation of the service that will receive incomming connections. +2. ``, mandatory, is the port number the upstream service is listening - this is not related to the listening port of HAProxy. +3. ``, optional, should be defined as `PROXY` if HAProxy should expect requests using the [PROXY](http://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) protocol. Leave empty to not use PROXY protocol. This is usually used only if there is another load balancer in front of HAProxy which supports the PROXY protocol. PROXY protocol v1 and v2 are supported. +4. ``, optional, should be defined as `PROXY` or `PROXY-V2` if the upstream service expect connections using the PROXY protocol v2. Use `PROXY-V1` instead if the upstream service only support v1 protocol. Leave empty to connect without using the PROXY protocol. +5. ``, optional, used to configure SSL/TLS over the TCP connection. Secret should have `tls.crt` and `tls.key` pair used on TLS handshake. Leave empty to not use ssl-offload. + +Optional fields should be skipped using two consecutive colons. In the example below: @@ -725,7 +728,7 @@ In the example below: data: "5432": "default/pgsql:5432" "8000": "system-prod/http:8000::PROXY-V1" - "9900": "system-prod/admin:9900:PROXY" + "9900": "system-prod/admin:9900:PROXY::system-prod/tcp-9900" "9990": "system-prod/admin:9999::PROXY-V2" "9999": "system-prod/admin:9999:PROXY:PROXY" ``` @@ -734,7 +737,7 @@ HAProxy will listen 5 new ports: * `5432` will proxy to a `pgsql` service on `default` namespace. * `8000` will proxy to `http` service, port `8000`, on the `system-prod` namespace. The upstream service will expect connections using the PROXY protocol but it only supports v1. -* `9900` will proxy to `admin` service, port `9900`, on the `system-prod` namespace. Clients should connect using the PROXY protocol v1 or v2. +* `9900` will proxy to `admin` service, port `9900`, on the `system-prod` namespace. Clients should connect using the PROXY protocol v1 or v2. Upcoming connections should be encrypted, HAProxy will ssl-offload data using crt/key provided by `system-prod/tcp-9900` secret. * `9990` and `9999` will proxy to the same `admin` service and `9999` port and the upstream service will expect connections using the PROXY protocol v2. The HAProxy frontend, however, will only expect PROXY protocol v1 or v2 on it's port `9999`. ### verify-hostname diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index 53d433d62..096280a79 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -362,12 +362,17 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1 continue } - nsSvcPort := utils.SplitMin(v, ":", 4) + // 1: namespace/name of the target service + // 2: port number + // 3: "PROXY" means accept proxy protocol + // 4: "PROXY[-V1|V2]" means send proxy protocol, defaults to V2 + // 5: namespace/name of crt/key secret if should ssl-offload + nsSvcPort := utils.SplitMin(v, ":", 5) nsName := nsSvcPort[0] svcPort := nsSvcPort[1] if nsName == "" || svcPort == "" { - glog.Warningf("invalid format (namespace/name:port:[PROXY]:[PROXY[-V1|-V2]]) '%v'", v) + glog.Warningf("invalid format (namespace/service-name:port:[PROXY]:[PROXY[-V1|-V2]]:namespace/secret-name) '%v'", v) continue } @@ -376,6 +381,17 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1 if proto == apiv1.ProtocolTCP { svcProxyProtocol.Decode = strings.ToUpper(nsSvcPort[2]) == "PROXY" svcProxyProtocol.EncodeVersion = proxyProtocolParamToVersion(nsSvcPort[3]) + } else if nsSvcPort[2] != "" || nsSvcPort[3] != "" { + glog.Warningf("ignoring PROXY protocol on non TCP service %v:%v", nsName, svcPort) + } + + crtSecret := nsSvcPort[4] + if crtSecret != "" { + _, _, err = k8s.ParseNameNS(crtSecret) + if err != nil { + glog.Warningf("%v", err) + continue + } } svcNs, svcName, err := k8s.ParseNameNS(nsName) @@ -397,6 +413,15 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1 svc := svcObj.(*apiv1.Service) + crt := &ingress.SSLCert{} + if crtSecret != "" { + crt, err = ic.GetCertificate(crtSecret) + if err != nil { + glog.Errorf("error reading crt/key of TCP service %v/%v: %v", nsName, svcPort, err) + continue + } + } + var endps []ingress.Endpoint targetPort, err := strconv.Atoi(svcPort) if err != nil { @@ -437,6 +462,7 @@ func (ic *GenericController) getStreamServices(configmapName string, proto apiv1 Port: intstr.FromString(svcPort), Protocol: proto, ProxyProtocol: svcProxyProtocol, + SSLCert: *crt, }, Endpoints: endps, }) diff --git a/pkg/common/ingress/types.go b/pkg/common/ingress/types.go index f453ca402..917208a94 100644 --- a/pkg/common/ingress/types.go +++ b/pkg/common/ingress/types.go @@ -412,6 +412,7 @@ type L4Backend struct { Protocol apiv1.Protocol `json:"protocol"` // +optional ProxyProtocol ProxyProtocol `json:"proxyProtocol"` + SSLCert SSLCert `json:"sslCert"` } // ProxyProtocol describes if the proxy protocol should be configured diff --git a/pkg/common/ingress/types_equals.go b/pkg/common/ingress/types_equals.go index f0bf62847..12d9da042 100644 --- a/pkg/common/ingress/types_equals.go +++ b/pkg/common/ingress/types_equals.go @@ -499,6 +499,9 @@ func (l4b1 *L4Backend) Equal(l4b2 *L4Backend) bool { if !l4b1.ProxyProtocol.Equal(&l4b2.ProxyProtocol) { return false } + if !l4b1.SSLCert.Equal(&l4b2.SSLCert) { + return false + } return true } diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index 8fb785d2b..4c163a577 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -85,7 +85,11 @@ userlist {{ $userlist.ListName }} listen tcp-{{ $tcp.Port }} {{- $inProxyProt := $tcp.Backend.ProxyProtocol.Decode }} {{- $outProxyProtVersion := $tcp.Backend.ProxyProtocol.EncodeVersion }} - bind {{ $cfg.BindIPAddrTCP }}:{{ $tcp.Port }}{{ if $inProxyProt }} accept-proxy{{ end }} +{{- $ssl := $tcp.Backend.SSLCert }} +{{- if ne $ssl.PemSHA "" }} + # CRT PEM checksum: {{ $ssl.PemSHA }} +{{- end }} + bind {{ $cfg.BindIPAddrTCP }}:{{ $tcp.Port }}{{ if ne $ssl.PemSHA "" }} ssl crt {{ $ssl.PemFileName }}{{ end }}{{ if $inProxyProt }} accept-proxy{{ end }} mode tcp {{- if ne $cfg.Syslog "" }} {{- if eq $cfg.TCPLogFormat "" }}