From 4c8640af36d079c2db99587bf3a78d18a91f9ae1 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 27 Jan 2019 14:29:31 -0200 Subject: [PATCH 1/8] add secret reading --- pkg/common/ingress/controller/controller.go | 28 ++++++++------- pkg/controller/cache.go | 39 ++++++++++++++++++--- pkg/controller/controller.go | 2 +- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index b3abd9d7d..c24a5a6e1 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -1540,19 +1540,21 @@ func (ic *GenericController) Start() { createDefaultSSLCertificate() - time.Sleep(5 * time.Second) - // initial sync of secrets to avoid unnecessary reloads - glog.Info("running initial sync of secrets") - for _, obj := range ic.listers.Ingress.List() { - ing := obj.(*extensions.Ingress) + if ic.cfg.V07 { + time.Sleep(5 * time.Second) + // initial sync of secrets to avoid unnecessary reloads + glog.Info("running initial sync of secrets") + for _, obj := range ic.listers.Ingress.List() { + ing := obj.(*extensions.Ingress) + + if !class.IsValid(ing, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) { + a, _ := parser.GetStringAnnotation(class.IngressKey, ing) + glog.V(2).Infof("ignoring add for ingress %v based on annotation %v with value %v", ing.Name, class.IngressKey, a) + continue + } - if !class.IsValid(ing, ic.cfg.IngressClass, ic.cfg.DefaultIngressClass) { - a, _ := parser.GetStringAnnotation(class.IngressKey, ing) - glog.V(2).Infof("ignoring add for ingress %v based on annotation %v with value %v", ing.Name, class.IngressKey, a) - continue + ic.readSecrets(ing) } - - ic.readSecrets(ing) } go ic.syncQueue.Run(time.Second, ic.stopCh) @@ -1561,7 +1563,9 @@ func (ic *GenericController) Start() { go ic.syncStatus.Run(ic.stopCh) } - go wait.Until(ic.checkMissingSecrets, 30*time.Second, ic.stopCh) + if ic.cfg.V07 { + go wait.Until(ic.checkMissingSecrets, 30*time.Second, ic.stopCh) + } // force initial sync ic.syncQueue.Enqueue(&extensions.Ingress{}) diff --git a/pkg/controller/cache.go b/pkg/controller/cache.go index eb7c9cff1..db2ee9eb4 100644 --- a/pkg/controller/cache.go +++ b/pkg/controller/cache.go @@ -23,10 +23,19 @@ import ( api "k8s.io/api/core/v1" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/controller" ) type cache struct { - listers *ingress.StoreLister + listers *ingress.StoreLister + controller *controller.GenericController +} + +func newCache(listers *ingress.StoreLister, controller *controller.GenericController) *cache { + return &cache{ + listers: listers, + controller: controller, + } } func (c *cache) GetService(serviceName string) (*api.Service, error) { @@ -47,13 +56,35 @@ func (c *cache) GetPod(podName string) (*api.Pod, error) { } func (c *cache) GetTLSSecretPath(secretName string) (string, error) { - return "", fmt.Errorf("implement") + sslCert, err := c.controller.GetCertificate(secretName) + if err != nil { + return "", err + } + if sslCert.PemFileName == "" { + return "", fmt.Errorf("secret '%s' does not have tls/key pair", secretName) + } + return sslCert.PemFileName, nil } func (c *cache) GetCASecretPath(secretName string) (string, error) { - return "", fmt.Errorf("implement") + sslCert, err := c.controller.GetCertificate(secretName) + if err != nil { + return "", err + } + if sslCert.CAFileName == "" { + return "", fmt.Errorf("secret '%s' does not have ca.crt key", secretName) + } + return sslCert.CAFileName, nil } func (c *cache) GetSecretContent(secretName, keyName string) ([]byte, error) { - return []byte{}, fmt.Errorf("implement") + secret, err := c.listers.Secret.GetByName(secretName) + if err != nil { + return nil, err + } + data, found := secret.Data[keyName] + if !found { + return nil, fmt.Errorf("secret '%s' does not have key '%s'", secretName, keyName) + } + return data, nil } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e6e18e68f..e31119503 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -98,7 +98,7 @@ func (hc *HAProxyController) configController() { logger := &logger{depth: 1} hc.converterOptions = &ingtypes.ConverterOptions{ Logger: logger, - Cache: &cache{listers: hc.storeLister}, + Cache: newCache(hc.storeLister, hc.controller), AnnotationPrefix: "ingress.kubernetes.io", DefaultBackend: hc.cfg.DefaultService, DefaultSSLSecret: hc.cfg.DefaultSSLCertificate, From dae2128ef1031458a8b8f2e82aa215b138979cec Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 27 Jan 2019 15:31:18 -0200 Subject: [PATCH 2/8] add global config parsing Global config is both configurations of global section from haproxy config file - eg nbproc, nbthread, DH param config, and also default values for frontends and backends - eg timeouts. Updater knows both sides - k8s objects or maps with configs from the user, and haproxy objects used to populate the templates. Legacy controller code is always used via interface - the only coupling is from `pkg/controller` code. --- pkg/common/ingress/controller/backend_ssl.go | 4 +- pkg/controller/cache.go | 23 +++- pkg/converters/ingress/annotations/backend.go | 10 +- .../ingress/annotations/backend_test.go | 6 +- .../ingress/annotations/frontend.go | 4 +- pkg/converters/ingress/annotations/global.go | 115 ++++++++++++++++++ pkg/converters/ingress/annotations/updater.go | 40 +++++- pkg/converters/ingress/defaults.go | 4 +- .../ingress/helper_test/cachemock.go | 9 ++ .../ingress/helper_test/updatermock.go | 41 +++++++ pkg/converters/ingress/ingress.go | 11 +- pkg/converters/ingress/ingress_test.go | 8 +- pkg/converters/ingress/types/annotations.go | 1 + pkg/converters/ingress/types/config.go | 4 +- pkg/converters/ingress/types/interfaces.go | 1 + pkg/haproxy/config.go | 10 +- pkg/haproxy/types/types.go | 78 ++++++++++++ 17 files changed, 341 insertions(+), 28 deletions(-) create mode 100644 pkg/converters/ingress/annotations/global.go create mode 100644 pkg/converters/ingress/helper_test/updatermock.go diff --git a/pkg/common/ingress/controller/backend_ssl.go b/pkg/common/ingress/controller/backend_ssl.go index 23b179a95..1d6a8e320 100644 --- a/pkg/common/ingress/controller/backend_ssl.go +++ b/pkg/common/ingress/controller/backend_ssl.go @@ -84,7 +84,9 @@ func (ic *GenericController) getPemCertificate(secret *apiv1.Secret) (*ingress.S ca := secret.Data["ca.crt"] // namespace/secretName -> namespace-secretName - nsSecName := strings.Replace(secretName, "/", "-", -1) + // use `_` instead if v0.8+ + sep := map[bool]string{true: "-", false: "_"} + nsSecName := strings.Replace(secretName, "/", sep[ic.cfg.V07], -1) var s *ingress.SSLCert var err error diff --git a/pkg/controller/cache.go b/pkg/controller/cache.go index db2ee9eb4..81aae20f2 100644 --- a/pkg/controller/cache.go +++ b/pkg/controller/cache.go @@ -24,6 +24,7 @@ import ( "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/controller" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/net/ssl" ) type cache struct { @@ -61,7 +62,7 @@ func (c *cache) GetTLSSecretPath(secretName string) (string, error) { return "", err } if sslCert.PemFileName == "" { - return "", fmt.Errorf("secret '%s' does not have tls/key pair", secretName) + return "", fmt.Errorf("secret '%s' does not have keys 'tls.crt' and 'tls.key'", secretName) } return sslCert.PemFileName, nil } @@ -72,11 +73,29 @@ func (c *cache) GetCASecretPath(secretName string) (string, error) { return "", err } if sslCert.CAFileName == "" { - return "", fmt.Errorf("secret '%s' does not have ca.crt key", secretName) + return "", fmt.Errorf("secret '%s' does not have key 'ca.crt'", secretName) } return sslCert.CAFileName, nil } +func (c *cache) GetDHSecretPath(secretName string) (string, error) { + secret, err := c.listers.Secret.GetByName(secretName) + if err != nil { + return "", err + } + dh, found := secret.Data[dhparamFilename] + if !found { + return "", fmt.Errorf("secret '%s' does not have key '%s'", secretName, dhparamFilename) + } + pem := strings.Replace(secretName, "/", "_", -1) + pemFileName, err := ssl.AddOrUpdateDHParam(pem, dh) + if err != nil { + return "", fmt.Errorf("error creating dh-param file '%s': %v", pem, err) + } + // file.SHA1(pemFileName) + return pemFileName, nil +} + func (c *cache) GetSecretContent(secretName, keyName string) ([]byte, error) { secret, err := c.listers.Secret.GetByName(secretName) if err != nil { diff --git a/pkg/converters/ingress/annotations/backend.go b/pkg/converters/ingress/annotations/backend.go index 5a3ced2ac..67e83b92f 100644 --- a/pkg/converters/ingress/annotations/backend.go +++ b/pkg/converters/ingress/annotations/backend.go @@ -25,7 +25,7 @@ import ( hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" ) -func (c *updater) buildAffinity(d *backData) { +func (c *updater) buildBackendAffinity(d *backData) { if d.ann.Affinity != "cookie" { if d.ann.Affinity != "" { c.logger.Error("unsupported affinity type on %v: %s", d.ann.Source, d.ann.Affinity) @@ -50,7 +50,7 @@ func (c *updater) buildAffinity(d *backData) { d.backend.Cookie.Key = d.ann.CookieKey } -func (c *updater) buildAuthHTTP(d *backData) { +func (c *updater) buildBackendAuthHTTP(d *backData) { if d.ann.AuthType != "basic" { if d.ann.AuthType != "" { c.logger.Error("unsupported authentication type on %v: %s", d.ann.Source, d.ann.AuthType) @@ -71,7 +71,7 @@ func (c *updater) buildAuthHTTP(d *backData) { return } userstr := string(userb) - users, errs := c.buildAuthHTTPExtractUserlist(d.ann.Source.Name, secretName, userstr) + users, errs := c.buildBackendAuthHTTPExtractUserlist(d.ann.Source.Name, secretName, userstr) for _, err := range errs { c.logger.Warn("ignoring malformed usr/passwd on secret '%s', declared on %v: %v", secretName, d.ann.Source, err) } @@ -83,7 +83,7 @@ func (c *updater) buildAuthHTTP(d *backData) { d.backend.HreqValidateUserlist(userlist) } -func (c *updater) buildAuthHTTPExtractUserlist(source, secret, users string) ([]hatypes.User, []error) { +func (c *updater) buildBackendAuthHTTPExtractUserlist(source, secret, users string) ([]hatypes.User, []error) { var userlist []hatypes.User var err []error for i, usr := range strings.Split(users, "\n") { @@ -125,7 +125,7 @@ func (c *updater) buildAuthHTTPExtractUserlist(source, secret, users string) ([] return userlist, err } -func (c *updater) buildBlueGreen(d *backData) { +func (c *updater) buildBackendBlueGreen(d *backData) { balance := d.ann.BlueGreenBalance if balance == "" { balance = d.ann.BlueGreenDeploy diff --git a/pkg/converters/ingress/annotations/backend_test.go b/pkg/converters/ingress/annotations/backend_test.go index ceb91cc6a..0c960a7b7 100644 --- a/pkg/converters/ingress/annotations/backend_test.go +++ b/pkg/converters/ingress/annotations/backend_test.go @@ -87,7 +87,7 @@ func TestAffinity(t *testing.T) { c := setup(t) u := c.createUpdater() d := c.createBackendData("default", "ing1", &test.ann) - u.buildAffinity(d) + u.buildBackendAffinity(d) if !reflect.DeepEqual(test.expCookie, d.backend.Cookie) { t.Errorf("config %d differs - expected: %+v - actual: %+v", i, test.expCookie, d.backend.Cookie) } @@ -208,7 +208,7 @@ usr2::clearpwd2`)}}, } c.cache.SecretContent = test.secrets d := c.createBackendData(test.namespace, test.ingname, &test.ann) - u.buildAuthHTTP(d) + u.buildBackendAuthHTTP(d) userlists := u.haproxy.Userlists() httpRequests := d.backend.HTTPRequests if len(userlists)+len(test.expUserlists) > 0 && !reflect.DeepEqual(test.expUserlists, userlists) { @@ -489,7 +489,7 @@ INFO-V(3) blue/green balance label 'v=3' on ingress 'default/ing1' does not refe d := c.createBackendData("default", "ing1", &test.ann) d.backend.Endpoints = test.endpoints u := c.createUpdater() - u.buildBlueGreen(d) + u.buildBackendBlueGreen(d) weights := make([]int, len(d.backend.Endpoints)) for j, ep := range d.backend.Endpoints { weights[j] = ep.Weight diff --git a/pkg/converters/ingress/annotations/frontend.go b/pkg/converters/ingress/annotations/frontend.go index 9ac6e1a3a..bda230c7a 100644 --- a/pkg/converters/ingress/annotations/frontend.go +++ b/pkg/converters/ingress/annotations/frontend.go @@ -16,7 +16,7 @@ limitations under the License. package annotations -func (c *updater) buildAuthTLS(d *frontData) { +func (c *updater) buildFrontendAuthTLS(d *frontData) { if d.ann.AuthTLSSecret == "" { return } @@ -27,7 +27,7 @@ func (c *updater) buildAuthTLS(d *frontData) { } } -func (c *updater) buildSSLPassthrough(d *frontData) { +func (c *updater) buildFrontendSSLPassthrough(d *frontData) { if !d.ann.SSLPassthrough { return } diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go new file mode 100644 index 000000000..1cde762ec --- /dev/null +++ b/pkg/converters/ingress/annotations/global.go @@ -0,0 +1,115 @@ +/* +Copyright 2019 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "fmt" + "strings" +) + +func (c *updater) buildGlobalProc(d *globalData) { + balance := d.config.NbprocBalance + if balance < 1 { + c.logger.Warn("invalid value of nbproc-balance configmap option (%v), using 1", balance) + balance = 1 + } + if balance > 1 { + // need to visit (at least) statistics and healthz bindings as well + // as admin socket before using more than one balance backend + c.logger.Warn("nbproc-balance configmap option (%v) greater than 1 is not yet supported, using 1", balance) + balance = 1 + } + ssl := d.config.NbprocSSL + if ssl < 0 { + c.logger.Warn("invalid value of nbproc-ssl configmap option (%v), using 0", ssl) + ssl = 0 + } + procs := balance + ssl + threads := d.config.Nbthread + if threads < 1 { + c.logger.Warn("invalid value of nbthread configmap option (%v), using 1", threads) + threads = 1 + } + bindprocBalance := "1" + if balance > 1 { + bindprocBalance = fmt.Sprintf("1-%v", balance) + } + bindprocSSL := "" + if ssl == 0 { + bindprocSSL = bindprocBalance + } else if ssl == 1 { + bindprocSSL = fmt.Sprintf("%v", balance+1) + } else if ssl > 1 { + bindprocSSL = fmt.Sprintf("%v-%v", balance+1, procs) + } + cpumap := "" + if threads > 1 { + if procs == 1 { + cpumap = fmt.Sprintf("auto:1/1-%v 0-%v", threads, threads-1) + } + } else if procs > 1 { + cpumap = fmt.Sprintf("auto:1-%v 0-%v", procs, procs-1) + } + d.global.Procs.Nbproc = procs + d.global.Procs.Nbthread = threads + d.global.Procs.NbprocBalance = balance + d.global.Procs.NbprocSSL = ssl + d.global.Procs.BindprocBalance = bindprocBalance + d.global.Procs.BindprocSSL = bindprocSSL + d.global.Procs.CPUMap = cpumap +} + +func (c *updater) buildGlobalTimeout(d *globalData) { + copyHAProxyTime(&d.global.Timeout.Client, d.config.TimeoutClient) + copyHAProxyTime(&d.global.Timeout.ClientFin, d.config.TimeoutClientFin) + copyHAProxyTime(&d.global.Timeout.Connect, d.config.TimeoutConnect) + copyHAProxyTime(&d.global.Timeout.HTTPRequest, d.config.TimeoutHTTPRequest) + copyHAProxyTime(&d.global.Timeout.KeepAlive, d.config.TimeoutKeepAlive) + copyHAProxyTime(&d.global.Timeout.Queue, d.config.TimeoutQueue) + copyHAProxyTime(&d.global.Timeout.Server, d.config.TimeoutServer) + copyHAProxyTime(&d.global.Timeout.ServerFin, d.config.TimeoutServerFin) + copyHAProxyTime(&d.global.Timeout.Tunnel, d.config.TimeoutTunnel) + copyHAProxyTime(&d.global.Timeout.Stop, d.config.TimeoutStop) +} + +func (c *updater) buildGlobalSSL(d *globalData) { + d.global.SSL.Ciphers = d.config.SSLCiphers + d.global.SSL.Options = d.config.SSLOptions + if d.config.SSLDHParam != "" { + if dhFilename, err := c.cache.GetDHSecretPath(d.config.SSLDHParam); err == nil { + d.global.SSL.DHParam.Filename = dhFilename + } else { + c.logger.Error("error reading DH params: %v", err) + } + } + d.global.SSL.DHParam.DefaultMaxSize = d.config.SSLDHDefaultMaxSize + d.global.SSL.Engine = d.config.SSLEngine + d.global.SSL.ModeAsync = d.config.SSLModeAsync +} + +func (c *updater) buildGlobalModSecurity(d *globalData) { + d.global.ModSecurity.Endpoints = strings.Split(d.config.ModsecurityEndpoints, ",") + d.global.ModSecurity.Timeout.Hello = d.config.ModsecurityTimeoutHello + d.global.ModSecurity.Timeout.Idle = d.config.ModsecurityTimeoutIdle + d.global.ModSecurity.Timeout.Processing = d.config.ModsecurityTimeoutProcessing +} + +func (c *updater) buildGlobalCustomConfig(d *globalData) { + if d.config.ConfigGlobal != "" { + d.global.CustomConfig = strings.Split(strings.TrimRight(d.config.ConfigGlobal, "\n"), "\n") + } +} diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go index b3695b796..fac3ba452 100644 --- a/pkg/converters/ingress/annotations/updater.go +++ b/pkg/converters/ingress/annotations/updater.go @@ -25,6 +25,7 @@ import ( // Updater ... type Updater interface { + UpdateGlobalConfig(global *hatypes.Global, config *ingtypes.Config) UpdateFrontendConfig(frontend *hatypes.Frontend, ann *ingtypes.FrontendAnnotations) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.BackendAnnotations) } @@ -44,6 +45,11 @@ type updater struct { logger types.Logger } +type globalData struct { + global *hatypes.Global + config *ingtypes.Config +} + type frontData struct { frontend *hatypes.Frontend ann *ingtypes.FrontendAnnotations @@ -54,6 +60,30 @@ type backData struct { ann *ingtypes.BackendAnnotations } +func copyHAProxyTime(dst *string, src string) { + // TODO validate + *dst = src +} + +func (c *updater) UpdateGlobalConfig(global *hatypes.Global, config *ingtypes.Config) { + data := &globalData{ + global: global, + config: config, + } + global.Syslog.Endpoint = config.SyslogEndpoint + global.Syslog.Format = config.SyslogFormat + global.Syslog.Tag = config.SyslogTag + global.MaxConn = config.MaxConnections + global.DrainSupport = config.DrainSupport + global.LoadServerState = config.LoadServerState + global.StatsSocket = "/var/run/haproxy-stats.sock" + c.buildGlobalProc(data) + c.buildGlobalTimeout(data) + c.buildGlobalSSL(data) + c.buildGlobalModSecurity(data) + c.buildGlobalCustomConfig(data) +} + func (c *updater) UpdateFrontendConfig(frontend *hatypes.Frontend, ann *ingtypes.FrontendAnnotations) { data := &frontData{ frontend: frontend, @@ -64,8 +94,8 @@ func (c *updater) UpdateFrontendConfig(frontend *hatypes.Frontend, ann *ingtypes frontend.Alias.AliasRegex = ann.ServerAliasRegex frontend.Timeout.Client = ann.TimeoutClient frontend.Timeout.ClientFin = ann.TimeoutClientFin - c.buildAuthTLS(data) - c.buildSSLPassthrough(data) + c.buildFrontendAuthTLS(data) + c.buildFrontendSSLPassthrough(data) } func (c *updater) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.BackendAnnotations) { @@ -78,7 +108,7 @@ func (c *updater) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.Ba backend.MaxconnServer = ann.MaxconnServer backend.ProxyBodySize = ann.ProxyBodySize backend.SSLRedirect = ann.SSLRedirect - c.buildAffinity(data) - c.buildAuthHTTP(data) - c.buildBlueGreen(data) + c.buildBackendAffinity(data) + c.buildBackendAuthHTTP(data) + c.buildBackendBlueGreen(data) } diff --git a/pkg/converters/ingress/defaults.go b/pkg/converters/ingress/defaults.go index 27fe90ece..c67343643 100644 --- a/pkg/converters/ingress/defaults.go +++ b/pkg/converters/ingress/defaults.go @@ -92,7 +92,9 @@ func createDefaults() *types.Config { StatsProxyProtocol: false, StatsSSLCert: "", StrictHost: true, - Syslog: "", + SyslogEndpoint: "", + SyslogFormat: "rfc5424", + SyslogTag: "ingress", TCPLogFormat: "", TimeoutStop: "", UseProxyProtocol: false, diff --git a/pkg/converters/ingress/helper_test/cachemock.go b/pkg/converters/ingress/helper_test/cachemock.go index ce5863247..5399cdd10 100644 --- a/pkg/converters/ingress/helper_test/cachemock.go +++ b/pkg/converters/ingress/helper_test/cachemock.go @@ -33,6 +33,7 @@ type CacheMock struct { PodList map[string]*api.Pod SecretTLSPath map[string]string SecretCAPath map[string]string + SecretDHPath map[string]string SecretContent SecretContent } @@ -82,6 +83,14 @@ func (c *CacheMock) GetCASecretPath(secretName string) (string, error) { return "", fmt.Errorf("secret not found: '%s'", secretName) } +// GetDHSecretPath ... +func (c *CacheMock) GetDHSecretPath(secretName string) (string, error) { + if path, found := c.SecretDHPath[secretName]; found { + return path, nil + } + return "", fmt.Errorf("secret not found: '%s'", secretName) +} + // GetSecretContent ... func (c *CacheMock) GetSecretContent(secretName, keyName string) ([]byte, error) { if content, found := c.SecretContent[secretName]; found { diff --git a/pkg/converters/ingress/helper_test/updatermock.go b/pkg/converters/ingress/helper_test/updatermock.go new file mode 100644 index 000000000..c94613697 --- /dev/null +++ b/pkg/converters/ingress/helper_test/updatermock.go @@ -0,0 +1,41 @@ +/* +Copyright 2019 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper_test + +import ( + ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" +) + +// UpdaterMock ... +type UpdaterMock struct{} + +// UpdateGlobalConfig ... +func (u *UpdaterMock) UpdateGlobalConfig(global *hatypes.Global, config *ingtypes.Config) { +} + +// UpdateFrontendConfig ... +func (u *UpdaterMock) UpdateFrontendConfig(frontend *hatypes.Frontend, ann *ingtypes.FrontendAnnotations) { + frontend.Timeout.Client = ann.TimeoutClient + frontend.RootRedirect = ann.AppRoot +} + +// UpdateBackendConfig ... +func (u *UpdaterMock) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.BackendAnnotations) { + backend.MaxconnServer = ann.MaxconnServer + backend.BalanceAlgorithm = ann.BalanceAlgorithm +} diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index 7fdc285d7..45be47170 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -43,6 +43,7 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con options: options, logger: options.Logger, cache: options.Cache, + updater: annotations.NewUpdater(haproxy, options.Cache, options.Logger), globalConfig: mergeConfig(createDefaults(), globalConfig), frontendAnnotations: map[string]*ingtypes.FrontendAnnotations{}, backendAnnotations: map[string]*ingtypes.BackendAnnotations{}, @@ -60,6 +61,7 @@ type converter struct { options *ingtypes.ConverterOptions logger types.Logger cache ingtypes.Cache + updater annotations.Updater globalConfig *ingtypes.Config frontendAnnotations map[string]*ingtypes.FrontendAnnotations backendAnnotations map[string]*ingtypes.BackendAnnotations @@ -139,15 +141,15 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { } func (c *converter) syncAnnotations() { - updater := annotations.NewUpdater(c.haproxy, c.cache, c.logger) + c.updater.UpdateGlobalConfig(c.haproxy.Global(), c.globalConfig) for _, frontend := range c.haproxy.Frontends() { if ann, found := c.frontendAnnotations[frontend.Hostname]; found { - updater.UpdateFrontendConfig(frontend, ann) + c.updater.UpdateFrontendConfig(frontend, ann) } } for _, backend := range c.haproxy.Backends() { if ann, found := c.backendAnnotations[backend.ID]; found { - updater.UpdateBackendConfig(backend, ann) + c.updater.UpdateBackendConfig(backend, ann) } } } @@ -212,7 +214,8 @@ func (c *converter) addBackend(fullSvcName string, svcPort int, ingAnn *ingtypes // Merging Ingress annotations skipped, _ := utils.UpdateStruct(c.globalConfig.ConfigDefaults, ingAnn, ann) if len(skipped) > 0 { - c.logger.Info("skipping backend annotation(s) from %v due to conflict: %v", ingAnn.Source, skipped) + c.logger.Info("skipping backend '%s/%s:%d' annotation(s) from %v due to conflict: %v", + backend.Namespace, backend.Name, backend.Port, ingAnn.Source, skipped) } return backend, nil } diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index ede548852..1fdbdc218 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -830,7 +830,7 @@ func TestSyncAnnBackSvcIngConflict(t *testing.T) { balancealgorithm: leastconn`) c.compareLogging(` -INFO skipping backend annotation(s) from ingress 'default/echo' due to conflict: [balance-algorithm]`) +INFO skipping backend 'default/echo:8080' annotation(s) from ingress 'default/echo' due to conflict: [balance-algorithm]`) } func TestSyncAnnBacksSvcIng(t *testing.T) { @@ -930,7 +930,7 @@ func TestSyncAnnBackDefault(t *testing.T) { balancealgorithm: leastconn`) c.compareLogging(` -INFO skipping backend annotation(s) from ingress 'default/echo5' due to conflict: [balance-algorithm]`) +INFO skipping backend 'default/echo5:8080' annotation(s) from ingress 'default/echo5' due to conflict: [balance-algorithm]`) } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * @@ -943,8 +943,9 @@ type testConfig struct { t *testing.T decode func(data []byte, defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) hconfig haproxy.Config - cache *ing_helper.CacheMock logger *types_helper.LoggerMock + cache *ing_helper.CacheMock + updater *ing_helper.UpdaterMock } func setup(t *testing.T) *testConfig { @@ -989,6 +990,7 @@ func (c *testConfig) SyncDef(config map[string]string, ing ...*extensions.Ingres c.hconfig, config, ).(*converter) + conv.updater = c.updater conv.globalConfig = mergeConfig(&ingtypes.Config{}, config) conv.Sync(ing) } diff --git a/pkg/converters/ingress/types/annotations.go b/pkg/converters/ingress/types/annotations.go index 6e4cf3b09..5621ad6b1 100644 --- a/pkg/converters/ingress/types/annotations.go +++ b/pkg/converters/ingress/types/annotations.go @@ -69,6 +69,7 @@ type BackendAnnotations struct { SecureBackends bool `json:"secure-backends"` SecureCrtSecret string `json:"secure-crt-secret"` SecureVerifyCASecret string `json:"secure-verify-ca-secret"` + SessionCookieDynamic string `json:"session-cookie-dynamic"` SessionCookieName string `json:"session-cookie-name"` SessionCookieStrategy string `json:"session-cookie-strategy"` SSLRedirect bool `json:"ssl-redirect"` diff --git a/pkg/converters/ingress/types/config.go b/pkg/converters/ingress/types/config.go index 79a023fdd..ff6ad1ee0 100644 --- a/pkg/converters/ingress/types/config.go +++ b/pkg/converters/ingress/types/config.go @@ -84,7 +84,9 @@ type ConfigGlobals struct { StatsProxyProtocol bool `json:"stats-proxy-protocol"` StatsSSLCert string `json:"stats-ssl-cert"` StrictHost bool `json:"strict-host"` - Syslog string `json:"syslog-endpoint"` + SyslogEndpoint string `json:"syslog-endpoint"` + SyslogFormat string `json:"syslog-format"` + SyslogTag string `json:"syslog-tag"` TCPLogFormat string `json:"tcp-log-format"` TimeoutStop string `json:"timeout-stop"` UseProxyProtocol bool `json:"use-proxy-protocol"` diff --git a/pkg/converters/ingress/types/interfaces.go b/pkg/converters/ingress/types/interfaces.go index 0f637cf9a..08a875876 100644 --- a/pkg/converters/ingress/types/interfaces.go +++ b/pkg/converters/ingress/types/interfaces.go @@ -27,5 +27,6 @@ type Cache interface { GetPod(podName string) (*api.Pod, error) GetTLSSecretPath(secretName string) (string, error) GetCASecretPath(secretName string) (string, error) + GetDHSecretPath(secretName string) (string, error) GetSecretContent(secretName, keyName string) ([]byte, error) } diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 9f04e852b..10461be64 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -34,6 +34,7 @@ type Config interface { AddUserlist(name string, users []hatypes.User) *hatypes.Userlist FindUserlist(name string) *hatypes.Userlist DefaultBackend() *hatypes.Backend + Global() *hatypes.Global Frontends() []*hatypes.Frontend Backends() []*hatypes.Backend Userlists() []*hatypes.Userlist @@ -41,6 +42,7 @@ type Config interface { } type config struct { + global *hatypes.Global frontends []*hatypes.Frontend backends []*hatypes.Backend userlists []*hatypes.Userlist @@ -48,7 +50,9 @@ type config struct { } func createConfig() Config { - return &config{} + return &config{ + global: &hatypes.Global{}, + } } func (c *config) AcquireFrontend(hostname string) *hatypes.Frontend { @@ -148,6 +152,10 @@ func (c *config) DefaultBackend() *hatypes.Backend { return c.defaultBackend } +func (c *config) Global() *hatypes.Global { + return c.global +} + func (c *config) Frontends() []*hatypes.Frontend { return c.frontends } diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 59f8e5728..51a3ceeb7 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -16,6 +16,73 @@ limitations under the License. package types +// Global ... +type Global struct { + Procs ProcsConfig + Syslog SyslogConfig + MaxConn int + Timeout TimeoutConfig + SSL SSLConfig + ModSecurity ModSecurityConfig + DrainSupport bool + LoadServerState bool + StatsSocket string + CustomConfig []string +} + +// ProcsConfig ... +type ProcsConfig struct { + Nbproc int + Nbthread int + NbprocBalance int + NbprocSSL int + BindprocBalance string + BindprocSSL string + CPUMap string +} + +// SyslogConfig ... +type SyslogConfig struct { + Endpoint string + Format string + Tag string +} + +// TimeoutConfig ... +type TimeoutConfig struct { + FrontendTimeoutConfig + BackendTimeoutConfig + Stop string +} + +// SSLConfig ... +type SSLConfig struct { + DHParam DHParamConfig + Ciphers string + Options string + Engine string + ModeAsync bool +} + +// DHParamConfig ... +type DHParamConfig struct { + Filename string + DefaultMaxSize int +} + +// ModSecurityConfig ... +type ModSecurityConfig struct { + Endpoints []string + Timeout ModSecurityTimeoutConfig +} + +// ModSecurityTimeoutConfig ... +type ModSecurityTimeoutConfig struct { + Hello string + Idle string + Processing string +} + // Frontend ... // // Wildcard `*` hostname is a catch all and will be used if no other hostname, @@ -91,6 +158,17 @@ type Endpoint struct { Target string } +// BackendTimeoutConfig ... +type BackendTimeoutConfig struct { + Connect string + HTTPRequest string + KeepAlive string + Queue string + Server string + ServerFin string + Tunnel string +} + // Cookie ... type Cookie struct { Name string From 9210b7fbdff1d3215ae9c34fec5c7cc996449dff Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Mon, 28 Jan 2019 22:24:35 -0200 Subject: [PATCH 3/8] add default frontend --- pkg/converters/ingress/ingress_test.go | 59 ++++++++++++++++---------- pkg/haproxy/config.go | 31 ++++++++++---- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index 1fdbdc218..1032a817f 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -504,11 +504,11 @@ func TestSyncBackendDefault(t *testing.T) { c.createSvc1Auto() c.Sync(c.createIng2("default/echo", "echo:8080")) - c.compareConfigFront(` -- hostname: '*' - paths: - - path: / - backend: default_echo_8080`) + c.compareConfigDefaultFront(` +hostname: '*' +paths: +- path: / + backend: default_echo_8080`) c.compareConfigBack(` - id: default_echo_8080 @@ -563,11 +563,11 @@ func TestSyncDefaultBackendReusedPath1(t *testing.T) { c.createIng2("default/echo2", "echo2:8080"), ) - c.compareConfigFront(` -- hostname: '*' - paths: - - path: / - backend: default_echo1_8080`) + c.compareConfigDefaultFront(` +hostname: '*' +paths: +- path: / + backend: default_echo1_8080`) c.compareConfigBack(` - id: default_echo1_8080 @@ -590,11 +590,11 @@ func TestSyncDefaultBackendReusedPath2(t *testing.T) { c.createIng1("default/echo2", "'*'", "/", "echo2:8080"), ) - c.compareConfigFront(` -- hostname: '*' - paths: - - path: / - backend: default_echo1_8080`) + c.compareConfigDefaultFront(` +hostname: '*' +paths: +- path: / + backend: default_echo1_8080`) c.compareConfigBack(` - id: default_echo1_8080 @@ -621,11 +621,11 @@ func TestSyncEmptyHost(t *testing.T) { c.createSvc1Auto() c.Sync(c.createIng1("default/echo", "", "/", "echo:8080")) - c.compareConfigFront(` -- hostname: '*' - paths: - - path: / - backend: default_echo_8080`) + c.compareConfigDefaultFront(` +hostname: '*' +paths: +- path: / + backend: default_echo_8080`) } func TestSyncMultiNamespace(t *testing.T) { @@ -1179,9 +1179,9 @@ type ( } ) -func (c *testConfig) compareConfigFront(expected string) { +func convertFrontend(hafronts ...*hatypes.Frontend) []frontendMock { frontends := []frontendMock{} - for _, f := range c.hconfig.Frontends() { + for _, f := range hafronts { paths := []pathMock{} for _, p := range f.Paths { paths = append(paths, pathMock{Path: p.Path, BackendID: p.BackendID}) @@ -1194,7 +1194,20 @@ func (c *testConfig) compareConfigFront(expected string) { TLS: tlsMock{TLSFilename: f.TLS.TLSFilename}, }) } - c.compareText(_yamlMarshal(frontends), expected) + return frontends +} + +func (c *testConfig) compareConfigFront(expected string) { + c.compareText(_yamlMarshal(convertFrontend(c.hconfig.Frontends()...)), expected) +} + +func (c *testConfig) compareConfigDefaultFront(expected string) { + frontend := c.hconfig.DefaultFrontend() + if frontend != nil { + c.compareText(_yamlMarshal(convertFrontend(frontend)[0]), expected) + } else { + c.compareText("[]", expected) + } } type ( diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 10461be64..6d69d9bd5 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -33,6 +33,7 @@ type Config interface { ConfigDefaultBackend(defaultBackend *hatypes.Backend) AddUserlist(name string, users []hatypes.User) *hatypes.Userlist FindUserlist(name string) *hatypes.Userlist + DefaultFrontend() *hatypes.Frontend DefaultBackend() *hatypes.Backend Global() *hatypes.Global Frontends() []*hatypes.Frontend @@ -42,11 +43,12 @@ type Config interface { } type config struct { - global *hatypes.Global - frontends []*hatypes.Frontend - backends []*hatypes.Backend - userlists []*hatypes.Userlist - defaultBackend *hatypes.Backend + global *hatypes.Global + frontends []*hatypes.Frontend + backends []*hatypes.Backend + userlists []*hatypes.Userlist + defaultFrontend *hatypes.Frontend + defaultBackend *hatypes.Backend } func createConfig() Config { @@ -60,14 +62,21 @@ func (c *config) AcquireFrontend(hostname string) *hatypes.Frontend { return frontend } frontend := createFrontend(hostname) - c.frontends = append(c.frontends, frontend) - sort.Slice(c.frontends, func(i, j int) bool { - return c.frontends[i].Hostname < c.frontends[j].Hostname - }) + if frontend.Hostname != "*" { + c.frontends = append(c.frontends, frontend) + sort.Slice(c.frontends, func(i, j int) bool { + return c.frontends[i].Hostname < c.frontends[j].Hostname + }) + } else { + c.defaultFrontend = frontend + } return frontend } func (c *config) FindFrontend(hostname string) *hatypes.Frontend { + if hostname == "*" && c.defaultFrontend != nil { + return c.defaultFrontend + } for _, f := range c.frontends { if f.Hostname == hostname { return f @@ -148,6 +157,10 @@ func (c *config) FindUserlist(name string) *hatypes.Userlist { return nil } +func (c *config) DefaultFrontend() *hatypes.Frontend { + return c.defaultFrontend +} + func (c *config) DefaultBackend() *hatypes.Backend { return c.defaultBackend } From ebc26225c7079fe3d30ff74a9da5b6a3668c773f Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 10 Feb 2019 20:27:43 -0200 Subject: [PATCH 4/8] rename from frontend to host --- .../annotations/{frontend.go => host.go} | 18 ++-- pkg/converters/ingress/annotations/updater.go | 30 +++---- .../ingress/helper_test/updatermock.go | 8 +- pkg/converters/ingress/ingress.go | 84 +++++++++---------- pkg/converters/ingress/ingress_test.go | 24 +++--- pkg/converters/ingress/types/annotations.go | 4 +- pkg/haproxy/config.go | 60 ++++++------- pkg/haproxy/config_test.go | 14 ++-- pkg/haproxy/types/{frontend.go => host.go} | 12 +-- pkg/haproxy/types/types.go | 32 +++---- 10 files changed, 143 insertions(+), 143 deletions(-) rename pkg/converters/ingress/annotations/{frontend.go => host.go} (74%) rename pkg/haproxy/types/{frontend.go => host.go} (74%) diff --git a/pkg/converters/ingress/annotations/frontend.go b/pkg/converters/ingress/annotations/host.go similarity index 74% rename from pkg/converters/ingress/annotations/frontend.go rename to pkg/converters/ingress/annotations/host.go index bda230c7a..15207e91c 100644 --- a/pkg/converters/ingress/annotations/frontend.go +++ b/pkg/converters/ingress/annotations/host.go @@ -16,35 +16,35 @@ limitations under the License. package annotations -func (c *updater) buildFrontendAuthTLS(d *frontData) { +func (c *updater) buildHostAuthTLS(d *hostData) { if d.ann.AuthTLSSecret == "" { return } if cafile, err := c.cache.GetCASecretPath(d.ann.AuthTLSSecret); err == nil { - d.frontend.TLS.CAFilename = cafile - d.frontend.TLS.ErrorPage = d.ann.AuthTLSErrorPage - d.frontend.TLS.AddCertHeader = d.ann.AuthTLSCertHeader + d.host.TLS.CAFilename = cafile + d.host.TLS.ErrorPage = d.ann.AuthTLSErrorPage + d.host.TLS.AddCertHeader = d.ann.AuthTLSCertHeader } } -func (c *updater) buildFrontendSSLPassthrough(d *frontData) { +func (c *updater) buildHostSSLPassthrough(d *hostData) { if !d.ann.SSLPassthrough { return } - rootPath := d.frontend.FindPath("/") + rootPath := d.host.FindPath("/") if rootPath == nil { c.logger.Warn("skipping SSL of %s: root path was not configured", d.ann.Source) return } - for _, path := range d.frontend.Paths { + for _, path := range d.host.Paths { if path.Path != "/" { c.logger.Warn("ignoring path '%s' from '%s': ssl-passthrough only support root path", path.Path, d.ann.Source) } } if d.ann.SSLPassthroughHTTPPort != 0 { httpBackend := c.haproxy.FindBackend(rootPath.Backend.Namespace, rootPath.Backend.Name, d.ann.SSLPassthroughHTTPPort) - d.frontend.HTTPPassthroughBackend = httpBackend + d.host.HTTPPassthroughBackend = httpBackend } rootPath.Backend.ModeTCP = true - d.frontend.SSLPassthrough = true + d.host.SSLPassthrough = true } diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go index fac3ba452..8f50e398c 100644 --- a/pkg/converters/ingress/annotations/updater.go +++ b/pkg/converters/ingress/annotations/updater.go @@ -26,7 +26,7 @@ import ( // Updater ... type Updater interface { UpdateGlobalConfig(global *hatypes.Global, config *ingtypes.Config) - UpdateFrontendConfig(frontend *hatypes.Frontend, ann *ingtypes.FrontendAnnotations) + UpdateHostConfig(host *hatypes.Host, ann *ingtypes.HostAnnotations) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.BackendAnnotations) } @@ -50,9 +50,9 @@ type globalData struct { config *ingtypes.Config } -type frontData struct { - frontend *hatypes.Frontend - ann *ingtypes.FrontendAnnotations +type hostData struct { + host *hatypes.Host + ann *ingtypes.HostAnnotations } type backData struct { @@ -84,18 +84,18 @@ func (c *updater) UpdateGlobalConfig(global *hatypes.Global, config *ingtypes.Co c.buildGlobalCustomConfig(data) } -func (c *updater) UpdateFrontendConfig(frontend *hatypes.Frontend, ann *ingtypes.FrontendAnnotations) { - data := &frontData{ - frontend: frontend, - ann: ann, +func (c *updater) UpdateHostConfig(host *hatypes.Host, ann *ingtypes.HostAnnotations) { + data := &hostData{ + host: host, + ann: ann, } - frontend.RootRedirect = ann.AppRoot - frontend.Alias.AliasName = ann.ServerAlias - frontend.Alias.AliasRegex = ann.ServerAliasRegex - frontend.Timeout.Client = ann.TimeoutClient - frontend.Timeout.ClientFin = ann.TimeoutClientFin - c.buildFrontendAuthTLS(data) - c.buildFrontendSSLPassthrough(data) + host.RootRedirect = ann.AppRoot + host.Alias.AliasName = ann.ServerAlias + host.Alias.AliasRegex = ann.ServerAliasRegex + host.Timeout.Client = ann.TimeoutClient + host.Timeout.ClientFin = ann.TimeoutClientFin + c.buildHostAuthTLS(data) + c.buildHostSSLPassthrough(data) } func (c *updater) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.BackendAnnotations) { diff --git a/pkg/converters/ingress/helper_test/updatermock.go b/pkg/converters/ingress/helper_test/updatermock.go index c94613697..e0ba00621 100644 --- a/pkg/converters/ingress/helper_test/updatermock.go +++ b/pkg/converters/ingress/helper_test/updatermock.go @@ -28,10 +28,10 @@ type UpdaterMock struct{} func (u *UpdaterMock) UpdateGlobalConfig(global *hatypes.Global, config *ingtypes.Config) { } -// UpdateFrontendConfig ... -func (u *UpdaterMock) UpdateFrontendConfig(frontend *hatypes.Frontend, ann *ingtypes.FrontendAnnotations) { - frontend.Timeout.Client = ann.TimeoutClient - frontend.RootRedirect = ann.AppRoot +// UpdateHostConfig ... +func (u *UpdaterMock) UpdateHostConfig(host *hatypes.Host, ann *ingtypes.HostAnnotations) { + host.Timeout.Client = ann.TimeoutClient + host.RootRedirect = ann.AppRoot } // UpdateBackendConfig ... diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index 45be47170..eca5d4dcf 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -39,14 +39,14 @@ type Config interface { // NewIngressConverter ... func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Config, globalConfig map[string]string) Config { c := &converter{ - haproxy: haproxy, - options: options, - logger: options.Logger, - cache: options.Cache, - updater: annotations.NewUpdater(haproxy, options.Cache, options.Logger), - globalConfig: mergeConfig(createDefaults(), globalConfig), - frontendAnnotations: map[string]*ingtypes.FrontendAnnotations{}, - backendAnnotations: map[string]*ingtypes.BackendAnnotations{}, + haproxy: haproxy, + options: options, + logger: options.Logger, + cache: options.Cache, + updater: annotations.NewUpdater(haproxy, options.Cache, options.Logger), + globalConfig: mergeConfig(createDefaults(), globalConfig), + hostAnnotations: map[string]*ingtypes.HostAnnotations{}, + backendAnnotations: map[string]*ingtypes.BackendAnnotations{}, } if backend, err := c.addBackend(options.DefaultBackend, 0, &ingtypes.BackendAnnotations{}); err == nil { haproxy.ConfigDefaultBackend(backend) @@ -57,14 +57,14 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con } type converter struct { - haproxy haproxy.Config - options *ingtypes.ConverterOptions - logger types.Logger - cache ingtypes.Cache - updater annotations.Updater - globalConfig *ingtypes.Config - frontendAnnotations map[string]*ingtypes.FrontendAnnotations - backendAnnotations map[string]*ingtypes.BackendAnnotations + haproxy haproxy.Config + options *ingtypes.ConverterOptions + logger types.Logger + cache ingtypes.Cache + updater annotations.Updater + globalConfig *ingtypes.Config + hostAnnotations map[string]*ingtypes.HostAnnotations + backendAnnotations map[string]*ingtypes.BackendAnnotations } func (c *converter) Sync(ingress []*extensions.Ingress) { @@ -96,13 +96,13 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { if hostname == "" { hostname = "*" } - frontend := c.addFrontend(hostname, ingFrontAnn) + host := c.addHost(hostname, ingFrontAnn) for _, path := range rule.HTTP.Paths { uri := path.Path if uri == "" { uri = "/" } - if frontend.FindPath(uri) != nil { + if host.FindPath(uri) != nil { c.logger.Warn("skipping redeclared path '%s' of ingress '%s'", uri, fullIngName) continue } @@ -113,18 +113,18 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { c.logger.Warn("skipping backend config of ingress '%s': %v", fullIngName, err) continue } - frontend.AddPath(backend, uri) + host.AddPath(backend, uri) c.addHTTPPassthrough(fullSvcName, ingFrontAnn, ingBackAnn) } for _, tls := range ing.Spec.TLS { - for _, host := range tls.Hosts { - if host == hostname { + for _, tlshost := range tls.Hosts { + if tlshost == hostname { tlsPath, err := c.addTLS(ing.Namespace, tls.SecretName) if err == nil { - if frontend.TLS.TLSFilename == "" { - frontend.TLS.TLSFilename = tlsPath - } else if frontend.TLS.TLSFilename != tlsPath { - err = fmt.Errorf("TLS of host '%s' was already assigned", frontend.Hostname) + if host.TLS.TLSFilename == "" { + host.TLS.TLSFilename = tlsPath + } else if host.TLS.TLSFilename != tlsPath { + err = fmt.Errorf("TLS of host '%s' was already assigned", host.Hostname) } } if err != nil { @@ -142,9 +142,9 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { func (c *converter) syncAnnotations() { c.updater.UpdateGlobalConfig(c.haproxy.Global(), c.globalConfig) - for _, frontend := range c.haproxy.Frontends() { - if ann, found := c.frontendAnnotations[frontend.Hostname]; found { - c.updater.UpdateFrontendConfig(frontend, ann) + for _, host := range c.haproxy.Hosts() { + if ann, found := c.hostAnnotations[host.Hostname]; found { + c.updater.UpdateHostConfig(host, ann) } } for _, backend := range c.haproxy.Backends() { @@ -154,8 +154,8 @@ func (c *converter) syncAnnotations() { } } -func (c *converter) addDefaultHostBackend(fullSvcName string, svcPort int, ingFrontAnn *ingtypes.FrontendAnnotations, ingBackAnn *ingtypes.BackendAnnotations) error { - if fr := c.haproxy.FindFrontend("*"); fr != nil { +func (c *converter) addDefaultHostBackend(fullSvcName string, svcPort int, ingFrontAnn *ingtypes.HostAnnotations, ingBackAnn *ingtypes.BackendAnnotations) error { + if fr := c.haproxy.FindHost("*"); fr != nil { if fr.FindPath("/") != nil { return fmt.Errorf("path / was already defined on default host") } @@ -164,22 +164,22 @@ func (c *converter) addDefaultHostBackend(fullSvcName string, svcPort int, ingFr if err != nil { return err } - frontend := c.addFrontend("*", ingFrontAnn) - frontend.AddPath(backend, "/") + host := c.addHost("*", ingFrontAnn) + host.AddPath(backend, "/") return nil } -func (c *converter) addFrontend(host string, ingAnn *ingtypes.FrontendAnnotations) *hatypes.Frontend { - frontend := c.haproxy.AcquireFrontend(host) - if ann, found := c.frontendAnnotations[frontend.Hostname]; found { +func (c *converter) addHost(hostname string, ingAnn *ingtypes.HostAnnotations) *hatypes.Host { + host := c.haproxy.AcquireHost(hostname) + if ann, found := c.hostAnnotations[host.Hostname]; found { skipped, _ := utils.UpdateStruct(c.globalConfig.ConfigDefaults, ingAnn, ann) if len(skipped) > 0 { - c.logger.Info("skipping frontend annotation(s) from %v due to conflict: %v", ingAnn.Source, skipped) + c.logger.Info("skipping host annotation(s) from %v due to conflict: %v", ingAnn.Source, skipped) } } else { - c.frontendAnnotations[frontend.Hostname] = ingAnn + c.hostAnnotations[host.Hostname] = ingAnn } - return frontend + return host } func (c *converter) addBackend(fullSvcName string, svcPort int, ingAnn *ingtypes.BackendAnnotations) (*hatypes.Backend, error) { @@ -220,7 +220,7 @@ func (c *converter) addBackend(fullSvcName string, svcPort int, ingAnn *ingtypes return backend, nil } -func (c *converter) addHTTPPassthrough(fullSvcName string, ingFrontAnn *ingtypes.FrontendAnnotations, ingBackAnn *ingtypes.BackendAnnotations) { +func (c *converter) addHTTPPassthrough(fullSvcName string, ingFrontAnn *ingtypes.HostAnnotations, ingBackAnn *ingtypes.BackendAnnotations) { // a very specific use case of pre-parsing annotations: // need to add a backend if ssl-passthrough-http-port assigned if ingFrontAnn.SSLPassthrough && ingFrontAnn.SSLPassthroughHTTPPort != 0 { @@ -269,7 +269,7 @@ func (c *converter) addEndpoints(svc *api.Service, servicePort int, backend *hat return nil } -func (c *converter) readAnnotations(source *ingtypes.Source, annotations map[string]string) (*ingtypes.FrontendAnnotations, *ingtypes.BackendAnnotations) { +func (c *converter) readAnnotations(source *ingtypes.Source, annotations map[string]string) (*ingtypes.HostAnnotations, *ingtypes.BackendAnnotations) { ann := make(map[string]string, len(annotations)) prefix := c.options.AnnotationPrefix + "/" for annName, annValue := range annotations { @@ -278,12 +278,12 @@ func (c *converter) readAnnotations(source *ingtypes.Source, annotations map[str ann[name] = annValue } } - frontAnn := &ingtypes.FrontendAnnotations{Source: *source} + frontAnn := &ingtypes.HostAnnotations{Source: *source} backAnn := &ingtypes.BackendAnnotations{Source: *source} utils.UpdateStruct(struct{}{}, c.globalConfig.ConfigDefaults, frontAnn) utils.UpdateStruct(struct{}{}, c.globalConfig.ConfigDefaults, backAnn) if err := utils.MergeMap(ann, frontAnn); err != nil { - c.logger.Error("error merging frontend annotations from %v: %v", source, err) + c.logger.Error("error merging host annotations from %v: %v", source, err) } if err := utils.MergeMap(ann, backAnn); err != nil { c.logger.Error("error merging backend annotations from %v: %v", source, err) diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index 1032a817f..55868feea 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -128,7 +128,7 @@ func TestSyncReuseBackend(t *testing.T) { port: 8080`) } -func TestSyncReuseFrontend(t *testing.T) { +func TestSyncReuseHost(t *testing.T) { c := setup(t) defer c.teardown() @@ -205,7 +205,7 @@ func TestSyncRootPathLast(t *testing.T) { backend: default_echo_8080`) } -func TestSyncFrontendSorted(t *testing.T) { +func TestSyncHostSorted(t *testing.T) { c := setup(t) defer c.teardown() @@ -709,7 +709,7 @@ func TestSyncAnnFrontsConflict(t *testing.T) { client: 1s`) c.compareLogging(` -INFO skipping frontend annotation(s) from ingress 'default/echo2' due to conflict: [timeout-client]`) +INFO skipping host annotation(s) from ingress 'default/echo2' due to conflict: [timeout-client]`) } func TestSyncAnnFronts(t *testing.T) { @@ -1170,7 +1170,7 @@ type ( tlsMock struct { TLSFilename string `yaml:",omitempty"` } - frontendMock struct { + hostMock struct { Hostname string Paths []pathMock RootRedirect string `yaml:",omitempty"` @@ -1179,14 +1179,14 @@ type ( } ) -func convertFrontend(hafronts ...*hatypes.Frontend) []frontendMock { - frontends := []frontendMock{} +func convertHost(hafronts ...*hatypes.Host) []hostMock { + hosts := []hostMock{} for _, f := range hafronts { paths := []pathMock{} for _, p := range f.Paths { paths = append(paths, pathMock{Path: p.Path, BackendID: p.BackendID}) } - frontends = append(frontends, frontendMock{ + hosts = append(hosts, hostMock{ Hostname: f.Hostname, Paths: paths, RootRedirect: f.RootRedirect, @@ -1194,17 +1194,17 @@ func convertFrontend(hafronts ...*hatypes.Frontend) []frontendMock { TLS: tlsMock{TLSFilename: f.TLS.TLSFilename}, }) } - return frontends + return hosts } func (c *testConfig) compareConfigFront(expected string) { - c.compareText(_yamlMarshal(convertFrontend(c.hconfig.Frontends()...)), expected) + c.compareText(_yamlMarshal(convertHost(c.hconfig.Hosts()...)), expected) } func (c *testConfig) compareConfigDefaultFront(expected string) { - frontend := c.hconfig.DefaultFrontend() - if frontend != nil { - c.compareText(_yamlMarshal(convertFrontend(frontend)[0]), expected) + host := c.hconfig.DefaultHost() + if host != nil { + c.compareText(_yamlMarshal(convertHost(host)[0]), expected) } else { c.compareText("[]", expected) } diff --git a/pkg/converters/ingress/types/annotations.go b/pkg/converters/ingress/types/annotations.go index 5621ad6b1..e57b6bbf3 100644 --- a/pkg/converters/ingress/types/annotations.go +++ b/pkg/converters/ingress/types/annotations.go @@ -16,8 +16,8 @@ limitations under the License. package types -// FrontendAnnotations ... -type FrontendAnnotations struct { +// HostAnnotations ... +type HostAnnotations struct { Source Source `json:"-"` AppRoot string `json:"app-root"` AuthTLSCertHeader bool `json:"auth-tls-cert-header"` diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 6d69d9bd5..9f6846d67 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -26,29 +26,29 @@ import ( // Config ... type Config interface { - AcquireFrontend(hostname string) *hatypes.Frontend - FindFrontend(hostname string) *hatypes.Frontend + AcquireHost(hostname string) *hatypes.Host + FindHost(hostname string) *hatypes.Host AcquireBackend(namespace, name string, port int) *hatypes.Backend FindBackend(namespace, name string, port int) *hatypes.Backend ConfigDefaultBackend(defaultBackend *hatypes.Backend) AddUserlist(name string, users []hatypes.User) *hatypes.Userlist FindUserlist(name string) *hatypes.Userlist - DefaultFrontend() *hatypes.Frontend + DefaultHost() *hatypes.Host DefaultBackend() *hatypes.Backend Global() *hatypes.Global - Frontends() []*hatypes.Frontend + Hosts() []*hatypes.Host Backends() []*hatypes.Backend Userlists() []*hatypes.Userlist Equals(other Config) bool } type config struct { - global *hatypes.Global - frontends []*hatypes.Frontend - backends []*hatypes.Backend - userlists []*hatypes.Userlist - defaultFrontend *hatypes.Frontend - defaultBackend *hatypes.Backend + global *hatypes.Global + hosts []*hatypes.Host + backends []*hatypes.Backend + userlists []*hatypes.Userlist + defaultHost *hatypes.Host + defaultBackend *hatypes.Backend } func createConfig() Config { @@ -57,27 +57,27 @@ func createConfig() Config { } } -func (c *config) AcquireFrontend(hostname string) *hatypes.Frontend { - if frontend := c.FindFrontend(hostname); frontend != nil { - return frontend +func (c *config) AcquireHost(hostname string) *hatypes.Host { + if host := c.FindHost(hostname); host != nil { + return host } - frontend := createFrontend(hostname) - if frontend.Hostname != "*" { - c.frontends = append(c.frontends, frontend) - sort.Slice(c.frontends, func(i, j int) bool { - return c.frontends[i].Hostname < c.frontends[j].Hostname + host := createHost(hostname) + if host.Hostname != "*" { + c.hosts = append(c.hosts, host) + sort.Slice(c.hosts, func(i, j int) bool { + return c.hosts[i].Hostname < c.hosts[j].Hostname }) } else { - c.defaultFrontend = frontend + c.defaultHost = host } - return frontend + return host } -func (c *config) FindFrontend(hostname string) *hatypes.Frontend { - if hostname == "*" && c.defaultFrontend != nil { - return c.defaultFrontend +func (c *config) FindHost(hostname string) *hatypes.Host { + if hostname == "*" && c.defaultHost != nil { + return c.defaultHost } - for _, f := range c.frontends { + for _, f := range c.hosts { if f.Hostname == hostname { return f } @@ -85,8 +85,8 @@ func (c *config) FindFrontend(hostname string) *hatypes.Frontend { return nil } -func createFrontend(hostname string) *hatypes.Frontend { - return &hatypes.Frontend{ +func createHost(hostname string) *hatypes.Host { + return &hatypes.Host{ Hostname: hostname, } } @@ -157,8 +157,8 @@ func (c *config) FindUserlist(name string) *hatypes.Userlist { return nil } -func (c *config) DefaultFrontend() *hatypes.Frontend { - return c.defaultFrontend +func (c *config) DefaultHost() *hatypes.Host { + return c.defaultHost } func (c *config) DefaultBackend() *hatypes.Backend { @@ -169,8 +169,8 @@ func (c *config) Global() *hatypes.Global { return c.global } -func (c *config) Frontends() []*hatypes.Frontend { - return c.frontends +func (c *config) Hosts() []*hatypes.Host { + return c.hosts } func (c *config) Backends() []*hatypes.Backend { diff --git a/pkg/haproxy/config_test.go b/pkg/haproxy/config_test.go index 23aad15b1..95730f826 100644 --- a/pkg/haproxy/config_test.go +++ b/pkg/haproxy/config_test.go @@ -20,10 +20,10 @@ import ( "testing" ) -func TestAcquireFrontendDiff(t *testing.T) { +func TestAcquireHostDiff(t *testing.T) { c := createConfig() - f1 := c.AcquireFrontend("h1") - f2 := c.AcquireFrontend("h2") + f1 := c.AcquireHost("h1") + f2 := c.AcquireHost("h2") if f1.Hostname != "h1" { t.Errorf("expected %v but was %v", "h1", f1.Hostname) } @@ -32,12 +32,12 @@ func TestAcquireFrontendDiff(t *testing.T) { } } -func TestAcquireFrontendSame(t *testing.T) { +func TestAcquireHostSame(t *testing.T) { c := createConfig() - f1 := c.AcquireFrontend("h1") - f2 := c.AcquireFrontend("h1") + f1 := c.AcquireHost("h1") + f2 := c.AcquireHost("h1") if f1 != f2 { - t.Errorf("expected same frontend but was different") + t.Errorf("expected same host but was different") } } diff --git a/pkg/haproxy/types/frontend.go b/pkg/haproxy/types/host.go similarity index 74% rename from pkg/haproxy/types/frontend.go rename to pkg/haproxy/types/host.go index a25301b65..a70646683 100644 --- a/pkg/haproxy/types/frontend.go +++ b/pkg/haproxy/types/host.go @@ -21,8 +21,8 @@ import ( ) // FindPath ... -func (f *Frontend) FindPath(path string) *FrontendPath { - for _, p := range f.Paths { +func (h *Host) FindPath(path string) *HostPath { + for _, p := range h.Paths { if p.Path == path { return p } @@ -31,13 +31,13 @@ func (f *Frontend) FindPath(path string) *FrontendPath { } // AddPath ... -func (f *Frontend) AddPath(backend *Backend, path string) { - f.Paths = append(f.Paths, &FrontendPath{ +func (h *Host) AddPath(backend *Backend, path string) { + h.Paths = append(h.Paths, &HostPath{ Path: path, Backend: *backend, BackendID: backend.ID, }) - sort.Slice(f.Paths, func(i, j int) bool { - return f.Paths[i].Path > f.Paths[j].Path + sort.Slice(h.Paths, func(i, j int) bool { + return h.Paths[i].Path > h.Paths[j].Path }) } diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 51a3ceeb7..e13df230f 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -50,7 +50,7 @@ type SyslogConfig struct { // TimeoutConfig ... type TimeoutConfig struct { - FrontendTimeoutConfig + HostTimeoutConfig BackendTimeoutConfig Stop string } @@ -83,49 +83,49 @@ type ModSecurityTimeoutConfig struct { Processing string } -// Frontend ... +// Host ... // // Wildcard `*` hostname is a catch all and will be used if no other hostname, // alias or regex matches the request. If wildcard hostname is not declared, // the default backend will be used. If the default backend is empty, // a default 404 page generated by HAProxy will be used. -type Frontend struct { +type Host struct { Hostname string - Paths []*FrontendPath + Paths []*HostPath RootRedirect string - Alias FrontendAliasConfig + Alias HostAliasConfig HTTPPassthroughBackend *Backend SSLPassthrough bool - Timeout FrontendTimeoutConfig - TLS FrontendTLSConfig + Timeout HostTimeoutConfig + TLS HostTLSConfig } -// FrontendPath ... +// HostPath ... // // Root context `/` path is a catch all and will be used if no other path -// matches the request on this frontend. If a root context path is not +// matches the request on this host. If a root context path is not // declared, the default backend will be used. If the default backend is // empty, a default 404 page generated by HAProxy will be used. -type FrontendPath struct { +type HostPath struct { Path string Backend Backend BackendID string } -// FrontendAliasConfig ... -type FrontendAliasConfig struct { +// HostAliasConfig ... +type HostAliasConfig struct { AliasName string AliasRegex string } -// FrontendTimeoutConfig ... -type FrontendTimeoutConfig struct { +// HostTimeoutConfig ... +type HostTimeoutConfig struct { Client string ClientFin string } -// FrontendTLSConfig ... -type FrontendTLSConfig struct { +// HostTLSConfig ... +type HostTLSConfig struct { TLSFilename string TLSFileSHA256Sum string CAFilename string From 0ecc1616514605773a0ae176d00726c009c661d5 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 10 Feb 2019 20:33:05 -0200 Subject: [PATCH 5/8] use pointer as the host/backend map key --- pkg/converters/ingress/ingress.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index eca5d4dcf..96fcb9805 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -45,8 +45,8 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con cache: options.Cache, updater: annotations.NewUpdater(haproxy, options.Cache, options.Logger), globalConfig: mergeConfig(createDefaults(), globalConfig), - hostAnnotations: map[string]*ingtypes.HostAnnotations{}, - backendAnnotations: map[string]*ingtypes.BackendAnnotations{}, + hostAnnotations: map[*hatypes.Host]*ingtypes.HostAnnotations{}, + backendAnnotations: map[*hatypes.Backend]*ingtypes.BackendAnnotations{}, } if backend, err := c.addBackend(options.DefaultBackend, 0, &ingtypes.BackendAnnotations{}); err == nil { haproxy.ConfigDefaultBackend(backend) @@ -63,8 +63,8 @@ type converter struct { cache ingtypes.Cache updater annotations.Updater globalConfig *ingtypes.Config - hostAnnotations map[string]*ingtypes.HostAnnotations - backendAnnotations map[string]*ingtypes.BackendAnnotations + hostAnnotations map[*hatypes.Host]*ingtypes.HostAnnotations + backendAnnotations map[*hatypes.Backend]*ingtypes.BackendAnnotations } func (c *converter) Sync(ingress []*extensions.Ingress) { @@ -143,12 +143,12 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { func (c *converter) syncAnnotations() { c.updater.UpdateGlobalConfig(c.haproxy.Global(), c.globalConfig) for _, host := range c.haproxy.Hosts() { - if ann, found := c.hostAnnotations[host.Hostname]; found { + if ann, found := c.hostAnnotations[host]; found { c.updater.UpdateHostConfig(host, ann) } } for _, backend := range c.haproxy.Backends() { - if ann, found := c.backendAnnotations[backend.ID]; found { + if ann, found := c.backendAnnotations[backend]; found { c.updater.UpdateBackendConfig(backend, ann) } } @@ -171,13 +171,13 @@ func (c *converter) addDefaultHostBackend(fullSvcName string, svcPort int, ingFr func (c *converter) addHost(hostname string, ingAnn *ingtypes.HostAnnotations) *hatypes.Host { host := c.haproxy.AcquireHost(hostname) - if ann, found := c.hostAnnotations[host.Hostname]; found { + if ann, found := c.hostAnnotations[host]; found { skipped, _ := utils.UpdateStruct(c.globalConfig.ConfigDefaults, ingAnn, ann) if len(skipped) > 0 { c.logger.Info("skipping host annotation(s) from %v due to conflict: %v", ingAnn.Source, skipped) } } else { - c.hostAnnotations[host.Hostname] = ingAnn + c.hostAnnotations[host] = ingAnn } return host } @@ -197,7 +197,7 @@ func (c *converter) addBackend(fullSvcName string, svcPort int, ingAnn *ingtypes svcPort = svc.Spec.Ports[0].TargetPort.IntValue() } backend := c.haproxy.AcquireBackend(namespace, svcName, svcPort) - ann, found := c.backendAnnotations[backend.ID] + ann, found := c.backendAnnotations[backend] if !found { // New backend, configure endpoints and svc annotations if err := c.addEndpoints(svc, svcPort, backend); err != nil { @@ -209,7 +209,7 @@ func (c *converter) addBackend(fullSvcName string, svcPort int, ingAnn *ingtypes Name: svcName, Type: "service", }, svc.Annotations) - c.backendAnnotations[backend.ID] = ann + c.backendAnnotations[backend] = ann } // Merging Ingress annotations skipped, _ := utils.UpdateStruct(c.globalConfig.ConfigDefaults, ingAnn, ann) From 56b0bf2fe30c4d6ea611d86ce8495a049570e3e6 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Fri, 8 Mar 2019 22:14:49 -0300 Subject: [PATCH 6/8] starting template implementation --- pkg/common/ingress/controller/launch.go | 29 +- pkg/controller/controller.go | 29 +- pkg/converters/ingress/annotations/backend.go | 4 +- .../ingress/annotations/backend_test.go | 10 +- pkg/converters/ingress/annotations/host.go | 2 +- pkg/converters/ingress/annotations/updater.go | 2 +- .../ingress/annotations/updater_test.go | 3 +- .../ingress/helper_test/updatermock.go | 2 +- pkg/converters/ingress/ingress.go | 10 +- pkg/converters/ingress/ingress_test.go | 73 +- pkg/haproxy/config.go | 222 ++++- pkg/haproxy/config_test.go | 64 +- pkg/haproxy/helper_test/bindutilsmock.go | 38 + pkg/haproxy/instance.go | 114 ++- pkg/haproxy/instance_test.go | 799 ++++++++++++++++++ pkg/haproxy/template/template.go | 48 +- pkg/haproxy/template/template_test.go | 30 +- pkg/haproxy/types/backend.go | 13 +- pkg/haproxy/types/frontend.go | 176 ++++ pkg/haproxy/types/frontend_test.go | 167 ++++ pkg/haproxy/types/host.go | 13 +- pkg/haproxy/types/interfaces.go | 22 + pkg/haproxy/types/types.go | 131 ++- rootfs/Dockerfile | 2 +- rootfs/etc/haproxy/maptemplate/map.tmpl | 11 + .../haproxy/modsecurity/spoe-modsecurity.tmpl | 20 + rootfs/etc/haproxy/template/haproxy.tmpl | 401 +++++++++ rootfs/usr/local/etc/haproxy/errors/404.http | 8 + 28 files changed, 2256 insertions(+), 187 deletions(-) create mode 100644 pkg/haproxy/helper_test/bindutilsmock.go create mode 100644 pkg/haproxy/instance_test.go create mode 100644 pkg/haproxy/types/frontend.go create mode 100644 pkg/haproxy/types/frontend_test.go create mode 100644 pkg/haproxy/types/interfaces.go create mode 100644 rootfs/etc/haproxy/maptemplate/map.tmpl create mode 100644 rootfs/etc/haproxy/modsecurity/spoe-modsecurity.tmpl create mode 100644 rootfs/etc/haproxy/template/haproxy.tmpl create mode 100644 rootfs/usr/local/etc/haproxy/errors/404.http diff --git a/pkg/common/ingress/controller/launch.go b/pkg/common/ingress/controller/launch.go index 403696ba5..6b137d0b3 100644 --- a/pkg/common/ingress/controller/launch.go +++ b/pkg/common/ingress/controller/launch.go @@ -148,7 +148,7 @@ func NewIngressController(backend ingress.Controller) *GenericController { glog.Infof("Watching for ingress class: %s", *ingressClass) } - if *defaultSvc == "" { + if *v07 && *defaultSvc == "" { glog.Fatalf("Please specify --default-backend-service") } @@ -157,19 +157,21 @@ func NewIngressController(backend ingress.Controller) *GenericController { handleFatalInitError(err) } - ns, name, err := k8s.ParseNameNS(*defaultSvc) - if err != nil { - glog.Fatalf("invalid format for service %v: %v", *defaultSvc, err) - } + if *defaultSvc != "" { + ns, name, err := k8s.ParseNameNS(*defaultSvc) + if err != nil { + glog.Fatalf("invalid format for service %v: %v", *defaultSvc, err) + } - _, err = kubeClient.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) - if err != nil { - if strings.Contains(err.Error(), "cannot get services in the namespace") { - glog.Fatalf("✖ It seems the cluster it is running with Authorization enabled (like RBAC) and there is no permissions for the ingress controller. Please check the configuration") + _, err = kubeClient.CoreV1().Services(ns).Get(name, metav1.GetOptions{}) + if err != nil { + if strings.Contains(err.Error(), "cannot get services in the namespace") { + glog.Fatalf("✖ It seems the cluster it is running with Authorization enabled (like RBAC) and there is no permissions for the ingress controller. Please check the configuration") + } + glog.Fatalf("no service with name %v found: %v", *defaultSvc, err) } - glog.Fatalf("no service with name %v found: %v", *defaultSvc, err) + glog.Infof("validated %v as the default backend", *defaultSvc) } - glog.Infof("validated %v as the default backend", *defaultSvc) if *publishSvc != "" { ns, name, err := k8s.ParseNameNS(*publishSvc) @@ -199,6 +201,11 @@ func NewIngressController(backend ingress.Controller) *GenericController { if err != nil { glog.Fatalf("no watchNamespace with name %v found: %v", *watchNamespace, err) } + } else { + _, err = kubeClient.CoreV1().Services("default").Get("kubernetes", metav1.GetOptions{}) + if err != nil { + glog.Fatalf("error connecting to the apiserver: %v", err) + } } if *rateLimitUpdate <= 0 { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e31119503..c74dc1bc6 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -108,8 +108,33 @@ func (hc *HAProxyController) configController() { ReloadCmd: "/haproxy-reload.sh", HAProxyConfigFile: "/etc/haproxy/haproxy.cfg", ReloadStrategy: *hc.reloadStrategy, + MaxOldConfigFiles: *hc.maxOldConfigFiles, } - hc.instance = haproxy.CreateInstance(logger, instanceOptions) + hc.instance = haproxy.CreateInstance(logger, hc, instanceOptions) + if err := hc.instance.ParseTemplates(); err != nil { + glog.Fatalf("error creating HAProxy instance: %v", err) + } +} + +// CreateX509CertsDir hard link files from certs to a single directory. +func (hc *HAProxyController) CreateX509CertsDir(bindName string, certs []string) (string, error) { + x509dir := "/var/haproxy/certs/" + bindName + if err := os.RemoveAll(x509dir); err != nil { + return "", err + } + if err := os.MkdirAll(x509dir, 0700); err != nil { + return "", err + } + for _, cert := range certs { + file, err := os.Stat(cert) + if err != nil { + return "", err + } + if err := os.Link(cert, x509dir+"/"+file.Name()); err != os.ErrExist { + return "", err + } + } + return x509dir, nil } // Stop shutdown the controller process @@ -223,7 +248,7 @@ func (hc *HAProxyController) SyncIngress(item interface{}) error { } converter := ingressconverter.NewIngressConverter( hc.converterOptions, - hc.instance.CreateConfig(), + hc.instance.Config(), globalConfig, ) converter.Sync(ingress) diff --git a/pkg/converters/ingress/annotations/backend.go b/pkg/converters/ingress/annotations/backend.go index 67e83b92f..f0c8c9791 100644 --- a/pkg/converters/ingress/annotations/backend.go +++ b/pkg/converters/ingress/annotations/backend.go @@ -168,7 +168,7 @@ func (c *updater) buildBackendBlueGreen(d *backData) { } for _, ep := range d.backend.Endpoints { hasLabel := false - if pod, err := c.cache.GetPod(ep.Target); err == nil { + if pod, err := c.cache.GetPod(ep.TargetRef); err == nil { for _, dw := range deployWeights { if label, found := pod.Labels[dw.labelName]; found { if label == dw.labelValue { @@ -181,7 +181,7 @@ func (c *updater) buildBackendBlueGreen(d *backData) { } } } else { - if ep.Target == "" { + if ep.TargetRef == "" { err = fmt.Errorf("endpoint does not reference a pod") } c.logger.Warn("endpoint '%s:%d' on %v was removed from balance: %v", ep.IP, ep.Port, d.ann.Source, err) diff --git a/pkg/converters/ingress/annotations/backend_test.go b/pkg/converters/ingress/annotations/backend_test.go index 0c960a7b7..681e33679 100644 --- a/pkg/converters/ingress/annotations/backend_test.go +++ b/pkg/converters/ingress/annotations/backend_test.go @@ -243,12 +243,12 @@ func TestBlueGreen(t *testing.T) { buildEndpoints := func(targets string) []*hatypes.Endpoint { ep := []*hatypes.Endpoint{} if targets != "" { - for _, target := range strings.Split(targets, ",") { + for _, targetRef := range strings.Split(targets, ",") { ep = append(ep, &hatypes.Endpoint{ - IP: "172.17.0.11", - Port: 8080, - Weight: 1, - Target: target, + IP: "172.17.0.11", + Port: 8080, + Weight: 1, + TargetRef: targetRef, }) } } diff --git a/pkg/converters/ingress/annotations/host.go b/pkg/converters/ingress/annotations/host.go index 15207e91c..6ed0b9084 100644 --- a/pkg/converters/ingress/annotations/host.go +++ b/pkg/converters/ingress/annotations/host.go @@ -22,7 +22,7 @@ func (c *updater) buildHostAuthTLS(d *hostData) { } if cafile, err := c.cache.GetCASecretPath(d.ann.AuthTLSSecret); err == nil { d.host.TLS.CAFilename = cafile - d.host.TLS.ErrorPage = d.ann.AuthTLSErrorPage + d.host.TLS.CAErrorPage = d.ann.AuthTLSErrorPage d.host.TLS.AddCertHeader = d.ann.AuthTLSCertHeader } } diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go index 8f50e398c..d3acd8aca 100644 --- a/pkg/converters/ingress/annotations/updater.go +++ b/pkg/converters/ingress/annotations/updater.go @@ -105,7 +105,7 @@ func (c *updater) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.Ba } // TODO check ModeTCP with HTTP annotations backend.BalanceAlgorithm = ann.BalanceAlgorithm - backend.MaxconnServer = ann.MaxconnServer + backend.MaxConnServer = ann.MaxconnServer backend.ProxyBodySize = ann.ProxyBodySize backend.SSLRedirect = ann.SSLRedirect c.buildBackendAffinity(data) diff --git a/pkg/converters/ingress/annotations/updater_test.go b/pkg/converters/ingress/annotations/updater_test.go index 4af6416c3..6cde98ace 100644 --- a/pkg/converters/ingress/annotations/updater_test.go +++ b/pkg/converters/ingress/annotations/updater_test.go @@ -22,6 +22,7 @@ import ( ing_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/helper_test" "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" + ha_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/helper_test" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" types_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/types/helper_test" ) @@ -43,7 +44,7 @@ func setup(t *testing.T) *testConfig { logger := &types_helper.LoggerMock{T: t} return &testConfig{ t: t, - haproxy: haproxy.CreateInstance(logger, haproxy.InstanceOptions{}).Config(), + haproxy: haproxy.CreateInstance(logger, &ha_helper.BindUtilsMock{}, haproxy.InstanceOptions{}).Config(), cache: &ing_helper.CacheMock{}, logger: logger, } diff --git a/pkg/converters/ingress/helper_test/updatermock.go b/pkg/converters/ingress/helper_test/updatermock.go index e0ba00621..423e6d8b2 100644 --- a/pkg/converters/ingress/helper_test/updatermock.go +++ b/pkg/converters/ingress/helper_test/updatermock.go @@ -36,6 +36,6 @@ func (u *UpdaterMock) UpdateHostConfig(host *hatypes.Host, ann *ingtypes.HostAnn // UpdateBackendConfig ... func (u *UpdaterMock) UpdateBackendConfig(backend *hatypes.Backend, ann *ingtypes.BackendAnnotations) { - backend.MaxconnServer = ann.MaxconnServer + backend.MaxConnServer = ann.MaxconnServer backend.BalanceAlgorithm = ann.BalanceAlgorithm } diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index 96fcb9805..b2f2fbd94 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -48,10 +48,12 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con hostAnnotations: map[*hatypes.Host]*ingtypes.HostAnnotations{}, backendAnnotations: map[*hatypes.Backend]*ingtypes.BackendAnnotations{}, } - if backend, err := c.addBackend(options.DefaultBackend, 0, &ingtypes.BackendAnnotations{}); err == nil { - haproxy.ConfigDefaultBackend(backend) - } else { - c.logger.Error("error reading default service: %v", err) + if options.DefaultBackend != "" { + if backend, err := c.addBackend(options.DefaultBackend, 0, &ingtypes.BackendAnnotations{}); err == nil { + haproxy.ConfigDefaultBackend(backend) + } else { + c.logger.Error("error reading default service: %v", err) + } } return c } diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index 55868feea..f0dd5a514 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -31,6 +31,7 @@ import ( ing_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/helper_test" ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" + ha_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/helper_test" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" types_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/types/helper_test" ) @@ -51,11 +52,7 @@ func TestSyncSvcNotFound(t *testing.T) { - hostname: echo.example.com paths: []`) - c.compareConfigDefaultBack(` -id: system_default_8080 -endpoints: -- ip: 172.17.0.99 - port: 8080`) + c.compareConfigBack(defaultBackendConfig) c.compareLogging(` WARN skipping backend config of ingress 'default/echo': service not found: 'default/notfound'`) @@ -81,8 +78,6 @@ func TestSyncDefaultSvcNotFound(t *testing.T) { - ip: 172.17.0.11 port: 8080`) - c.compareConfigDefaultBack(`[]`) - c.compareLogging(` ERROR error reading default service: service not found: 'system/default'`) } @@ -106,7 +101,7 @@ func TestSyncSingle(t *testing.T) { - ip: 172.17.0.11 port: 8080 - ip: 172.17.0.28 - port: 8080`) + port: 8080` + defaultBackendConfig) } func TestSyncReuseBackend(t *testing.T) { @@ -125,7 +120,7 @@ func TestSyncReuseBackend(t *testing.T) { - ip: 172.17.0.10 port: 8080 - ip: 172.17.0.11 - port: 8080`) + port: 8080` + defaultBackendConfig) } func TestSyncReuseHost(t *testing.T) { @@ -162,7 +157,7 @@ func TestSyncNoEndpoint(t *testing.T) { backend: default_echo_8080`) c.compareConfigBack(` -- id: default_echo_8080`) +- id: default_echo_8080` + defaultBackendConfig) } func TestSyncInvalidEndpoint(t *testing.T) { @@ -180,7 +175,7 @@ func TestSyncInvalidEndpoint(t *testing.T) { backend: default_echo_8080`) c.compareConfigBack(` -- id: default_echo_8080`) +- id: default_echo_8080` + defaultBackendConfig) c.compareLogging(` ERROR error adding endpoints of service 'default/echo': could not find endpoints for service 'default/echo'`) @@ -258,7 +253,7 @@ func TestSyncBackendSorted(t *testing.T) { - id: default_echo3_8080 endpoints: - ip: 172.17.0.13 - port: 8080`) + port: 8080` + defaultBackendConfig) } func TestSyncRedeclarePath(t *testing.T) { @@ -282,7 +277,7 @@ func TestSyncRedeclarePath(t *testing.T) { - id: default_echo1_8080 endpoints: - ip: 172.17.0.11 - port: 8080`) + port: 8080` + defaultBackendConfig) c.compareLogging(` WARN skipping redeclared path '/p1' of ingress 'default/echo1'`) @@ -514,7 +509,7 @@ paths: - id: default_echo_8080 endpoints: - ip: 172.17.0.11 - port: 8080`) + port: 8080` + defaultBackendConfig) } func TestSyncBackendSvcNotFound(t *testing.T) { @@ -525,7 +520,7 @@ func TestSyncBackendSvcNotFound(t *testing.T) { c.Sync(c.createIng2("default/echo", "notfound:8080")) c.compareConfigFront(`[]`) - c.compareConfigBack(`[]`) + c.compareConfigBack(defaultBackendConfig) c.compareLogging(` WARN skipping default backend of ingress 'default/echo': service not found: 'default/notfound'`) @@ -541,15 +536,10 @@ func TestSyncBackendReuseDefaultSvc(t *testing.T) { - hostname: default.example.com paths: - path: /app - backend: system_default_8080`) - - c.compareConfigBack(`[]`) + backend: _default_backend`) - c.compareConfigDefaultBack(` -id: system_default_8080 -endpoints: -- ip: 172.17.0.99 - port: 8080`) + c.compareConfigDefaultFront(`[]`) + c.compareConfigBack(defaultBackendConfig) } func TestSyncDefaultBackendReusedPath1(t *testing.T) { @@ -573,7 +563,7 @@ paths: - id: default_echo1_8080 endpoints: - ip: 172.17.0.11 - port: 8080`) + port: 8080` + defaultBackendConfig) c.compareLogging(` WARN skipping default backend of ingress 'default/echo2': path / was already defined on default host`) @@ -600,7 +590,7 @@ paths: - id: default_echo1_8080 endpoints: - ip: 172.17.0.11 - port: 8080`) + port: 8080` + defaultBackendConfig) c.compareLogging(` WARN skipping redeclared path '/' of ingress 'default/echo2'`) @@ -656,7 +646,7 @@ func TestSyncMultiNamespace(t *testing.T) { - id: ns2_echo_8080 endpoints: - ip: 172.17.0.12 - port: 8080`) + port: 8080` + defaultBackendConfig) } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * @@ -791,7 +781,7 @@ func TestSyncAnnBack(t *testing.T) { endpoints: - ip: 172.17.0.11 port: 8080 - balancealgorithm: leastconn`) + balancealgorithm: leastconn` + defaultBackendConfig) } func TestSyncAnnBackSvc(t *testing.T) { @@ -808,7 +798,7 @@ func TestSyncAnnBackSvc(t *testing.T) { endpoints: - ip: 172.17.0.11 port: 8080 - balancealgorithm: leastconn`) + balancealgorithm: leastconn` + defaultBackendConfig) } func TestSyncAnnBackSvcIngConflict(t *testing.T) { @@ -827,7 +817,7 @@ func TestSyncAnnBackSvcIngConflict(t *testing.T) { endpoints: - ip: 172.17.0.11 port: 8080 - balancealgorithm: leastconn`) + balancealgorithm: leastconn` + defaultBackendConfig) c.compareLogging(` INFO skipping backend 'default/echo:8080' annotation(s) from ingress 'default/echo' due to conflict: [balance-algorithm]`) @@ -850,7 +840,7 @@ func TestSyncAnnBacksSvcIng(t *testing.T) { - ip: 172.17.0.11 port: 8080 balancealgorithm: leastconn - maxconnserver: 10`) + maxconnserver: 10` + defaultBackendConfig) } func TestSyncAnnBackDefault(t *testing.T) { @@ -927,7 +917,7 @@ func TestSyncAnnBackDefault(t *testing.T) { endpoints: - ip: 172.17.0.17 port: 8080 - balancealgorithm: leastconn`) + balancealgorithm: leastconn` + defaultBackendConfig) c.compareLogging(` INFO skipping backend 'default/echo5:8080' annotation(s) from ingress 'default/echo5' due to conflict: [balance-algorithm]`) @@ -956,7 +946,7 @@ func setup(t *testing.T) *testConfig { c := &testConfig{ t: t, decode: scheme.Codecs.UniversalDeserializer().Decode, - hconfig: haproxy.CreateInstance(logger, haproxy.InstanceOptions{}).Config(), + hconfig: haproxy.CreateInstance(logger, &ha_helper.BindUtilsMock{}, haproxy.InstanceOptions{}).Config(), cache: &ing_helper.CacheMock{ SvcList: []*api.Service{}, EpList: map[string]*api.Endpoints{}, @@ -978,6 +968,12 @@ func (c *testConfig) Sync(ing ...*extensions.Ingress) { c.SyncDef(map[string]string{}, ing...) } +var defaultBackendConfig = ` +- id: _default_backend + endpoints: + - ip: 172.17.0.99 + port: 8080` + func (c *testConfig) SyncDef(config map[string]string, ing ...*extensions.Ingress) { conv := NewIngressConverter( &ingtypes.ConverterOptions{ @@ -1219,7 +1215,7 @@ type ( ID string Endpoints []endpointMock `yaml:",omitempty"` BalanceAlgorithm string `yaml:",omitempty"` - MaxconnServer int `yaml:",omitempty"` + MaxConnServer int `yaml:",omitempty"` } ) @@ -1234,7 +1230,7 @@ func convertBackend(habackends ...*hatypes.Backend) []backendMock { ID: b.ID, Endpoints: endpoints, BalanceAlgorithm: b.BalanceAlgorithm, - MaxconnServer: b.MaxconnServer, + MaxConnServer: b.MaxConnServer, }) } return backends @@ -1244,15 +1240,6 @@ func (c *testConfig) compareConfigBack(expected string) { c.compareText(_yamlMarshal(convertBackend(c.hconfig.Backends()...)), expected) } -func (c *testConfig) compareConfigDefaultBack(expected string) { - backend := c.hconfig.DefaultBackend() - if backend != nil { - c.compareText(_yamlMarshal(convertBackend(backend)[0]), expected) - } else { - c.compareText("[]", expected) - } -} - func (c *testConfig) compareLogging(expected string) { c.compareText(strings.Join(c.logger.Logging, "\n"), expected) c.logger.Logging = []string{} diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 9f6846d67..97b418077 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -21,6 +21,7 @@ import ( "reflect" "sort" + "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/template" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" ) @@ -31,8 +32,10 @@ type Config interface { AcquireBackend(namespace, name string, port int) *hatypes.Backend FindBackend(namespace, name string, port int) *hatypes.Backend ConfigDefaultBackend(defaultBackend *hatypes.Backend) + ConfigDefaultX509Cert(filename string) AddUserlist(name string, users []hatypes.User) *hatypes.Userlist FindUserlist(name string) *hatypes.Userlist + BuildFrontendGroup() (*hatypes.FrontendGroup, error) DefaultHost() *hatypes.Host DefaultBackend() *hatypes.Backend Global() *hatypes.Global @@ -43,17 +46,33 @@ type Config interface { } type config struct { - global *hatypes.Global - hosts []*hatypes.Host - backends []*hatypes.Backend - userlists []*hatypes.Userlist - defaultHost *hatypes.Host - defaultBackend *hatypes.Backend + bindUtils hatypes.BindUtils + mapsTemplate *template.Config + mapsDir string + global *hatypes.Global + hosts []*hatypes.Host + backends []*hatypes.Backend + userlists []*hatypes.Userlist + defaultHost *hatypes.Host + defaultBackend *hatypes.Backend + defaultX509Cert string } -func createConfig() Config { +type options struct { + mapsTemplate *template.Config + mapsDir string +} + +func createConfig(bindUtils hatypes.BindUtils, options options) *config { + mapsTemplate := options.mapsTemplate + if mapsTemplate == nil { + mapsTemplate = template.CreateConfig() + } return &config{ - global: &hatypes.Global{}, + bindUtils: bindUtils, + global: &hatypes.Global{}, + mapsTemplate: mapsTemplate, + mapsDir: options.mapsDir, } } @@ -91,23 +110,29 @@ func createHost(hostname string) *hatypes.Host { } } +func (c *config) sortBackends() { + sort.Slice(c.backends, func(i, j int) bool { + if c.backends[i] == c.defaultBackend { + return false + } + if c.backends[j] == c.defaultBackend { + return true + } + return c.backends[i].ID < c.backends[j].ID + }) +} + func (c *config) AcquireBackend(namespace, name string, port int) *hatypes.Backend { if backend := c.FindBackend(namespace, name, port); backend != nil { return backend } backend := createBackend(namespace, name, port) c.backends = append(c.backends, backend) - sort.Slice(c.backends, func(i, j int) bool { - return c.backends[i].ID < c.backends[j].ID - }) + c.sortBackends() return backend } func (c *config) FindBackend(namespace, name string, port int) *hatypes.Backend { - // TODO test missing `== port` - if c.defaultBackend != nil && c.defaultBackend.Namespace == namespace && c.defaultBackend.Name == name { - return c.defaultBackend - } for _, b := range c.backends { if b.Namespace == namespace && b.Name == name && b.Port == port { return b @@ -131,14 +156,19 @@ func buildID(namespace, name string, port int) string { } func (c *config) ConfigDefaultBackend(defaultBackend *hatypes.Backend) { + if c.defaultBackend != nil { + def := c.defaultBackend + def.ID = buildID(def.Namespace, def.Name, def.Port) + } c.defaultBackend = defaultBackend - // remove the default backend from the list - for i, backend := range c.backends { - if backend.ID == defaultBackend.ID { - c.backends = append(c.backends[:i], c.backends[i+1:]...) - break - } + if c.defaultBackend != nil { + c.defaultBackend.ID = "_default_backend" } + c.sortBackends() +} + +func (c *config) ConfigDefaultX509Cert(filename string) { + c.defaultX509Cert = filename } func (c *config) AddUserlist(name string, users []hatypes.User) *hatypes.Userlist { @@ -157,6 +187,156 @@ func (c *config) FindUserlist(name string) *hatypes.Userlist { return nil } +func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { + if len(c.hosts) == 0 { + return nil, fmt.Errorf("cannot create frontends without hosts") + } + frontends, sslpassthrough := hatypes.BuildRawFrontends(c.hosts) + for _, frontend := range frontends { + prefix := c.mapsDir + "/" + frontend.Name + frontend.BackendsMap = prefix + ".map" + frontend.TLSInvalidCrtErrorPagesMap = prefix + "_inv_crt.map" + frontend.TLSNoCrtErrorPagesMap = prefix + "_no_crt.map" + frontend.VarNamespaceMap = prefix + "_k8s_ns.map" + } + fgroup := &hatypes.FrontendGroup{ + Frontends: frontends, + HasSSLPassthrough: len(sslpassthrough) > 0, + HTTPFrontsMap: c.mapsDir + "/http-front.map", + SSLPassthroughMap: c.mapsDir + "/sslpassthrough.map", + } + if fgroup.HasTCPProxy() { + // More than one HAProxy's frontend or bind, or using ssl-passthrough config, + // so need a `mode tcp` frontend with `inspect-delay` and `req.ssl_sni` + var i int + for _, frontend := range frontends { + for _, bind := range frontend.Binds { + var bindName string + if len(bind.Hosts) == 1 { + bindName = bind.Hosts[0].Hostname + bind.TLS.TLSCert = bind.Hosts[0].TLS.TLSFilename + } else { + i++ + bindName = fmt.Sprintf("_socket%03d", i) + x509dir, err := c.createCertsDir(bindName, bind.Hosts) + if err != nil { + return nil, err + } + bind.TLS.TLSCert = c.defaultX509Cert + bind.TLS.TLSCertDir = x509dir + } + bind.Name = bindName + bind.Socket = fmt.Sprintf("unix@/var/run/front_%s.sock", bindName) + bind.AcceptProxy = true + } + } + } else { + // One single HAProxy's frontend and bind + bind := frontends[0].Binds[0] + bind.Name = "_public" + bind.Socket = ":443" + if len(bind.Hosts) == 1 { + bind.TLS.TLSCert = bind.Hosts[0].TLS.TLSFilename + } else { + x509dir, err := c.createCertsDir(bind.Name, bind.Hosts) + if err != nil { + return nil, err + } + tls := &frontends[0].Binds[0].TLS + tls.TLSCert = c.defaultX509Cert + tls.TLSCertDir = x509dir + } + } + type mapEntry struct { + Key string + Value string + } + var sslpassthroughMap []mapEntry + var httpFront []mapEntry + for _, sslpassHost := range sslpassthrough { + rootPath := sslpassHost.FindPath("/") + if rootPath == nil { + return nil, fmt.Errorf("missing root path on host %s", sslpassHost.Hostname) + } + sslpassthroughMap = append(sslpassthroughMap, mapEntry{ + Key: sslpassHost.Hostname, + Value: rootPath.BackendID, + }) + if sslpassHost.HTTPPassthroughBackend != nil { + httpFront = append(httpFront, mapEntry{ + Key: sslpassHost.Hostname + "/", + Value: sslpassHost.HTTPPassthroughBackend.ID, + }) + } else { + fgroup.HasRedirectHTTPS = true + } + } + for _, f := range frontends { + var backendsMap []mapEntry + var invalidCrtMap []mapEntry + var noCrtMap []mapEntry + var varNamespaceMap []mapEntry + for _, host := range f.Hosts { + for _, path := range host.Paths { + entry := mapEntry{ + Key: host.Hostname + path.Path, + Value: path.BackendID, + } + backendsMap = append(backendsMap, entry) + if path.Backend.SSLRedirect { + fgroup.HasRedirectHTTPS = true + } else { + httpFront = append(httpFront, entry) + } + if host.VarNamespace { + entry.Value = path.Backend.Namespace + } else { + entry.Value = "-" + } + varNamespaceMap = append(varNamespaceMap, entry) + } + if host.HasTLSAuth() && host.TLS.CAErrorPage != "" { + entry := mapEntry{ + Key: host.Hostname, + Value: host.TLS.CAErrorPage, + } + invalidCrtMap = append(invalidCrtMap, entry) + noCrtMap = append(noCrtMap, entry) + } + } + if err := c.mapsTemplate.WriteOutput(backendsMap, f.BackendsMap); err != nil { + return nil, err + } + if err := c.mapsTemplate.WriteOutput(invalidCrtMap, f.TLSInvalidCrtErrorPagesMap); err != nil { + return nil, err + } + if err := c.mapsTemplate.WriteOutput(noCrtMap, f.TLSNoCrtErrorPagesMap); err != nil { + return nil, err + } + if err := c.mapsTemplate.WriteOutput(varNamespaceMap, f.VarNamespaceMap); err != nil { + return nil, err + } + } + if err := c.mapsTemplate.WriteOutput(sslpassthroughMap, fgroup.SSLPassthroughMap); err != nil { + return nil, err + } + if err := c.mapsTemplate.WriteOutput(httpFront, fgroup.HTTPFrontsMap); err != nil { + return nil, err + } + fgroup.HasHTTPHost = len(httpFront) > 0 + return fgroup, nil +} + +func (c *config) createCertsDir(bindName string, hosts []*hatypes.Host) (string, error) { + certs := make([]string, 0, len(hosts)) + for _, host := range hosts { + if host.TLS.TLSFilename != "" { + certs = append(certs, host.TLS.TLSFilename) + } + } + return c.bindUtils.CreateX509CertsDir(bindName, certs) +} + func (c *config) DefaultHost() *hatypes.Host { return c.defaultHost } diff --git a/pkg/haproxy/config_test.go b/pkg/haproxy/config_test.go index 95730f826..dba68dcfe 100644 --- a/pkg/haproxy/config_test.go +++ b/pkg/haproxy/config_test.go @@ -18,10 +18,23 @@ package haproxy import ( "testing" + + ha_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/helper_test" ) +func TestEmptyFrontend(t *testing.T) { + c := createConfig(&ha_helper.BindUtilsMock{}, options{}) + if _, err := c.BuildFrontendGroup(); err == nil { + t.Error("expected error creating empty frontend") + } + c.AcquireHost("empty") + if _, err := c.BuildFrontendGroup(); err != nil { + t.Errorf("error creating frontends: %v", err) + } +} + func TestAcquireHostDiff(t *testing.T) { - c := createConfig() + c := createConfig(&ha_helper.BindUtilsMock{}, options{}) f1 := c.AcquireHost("h1") f2 := c.AcquireHost("h2") if f1.Hostname != "h1" { @@ -33,7 +46,7 @@ func TestAcquireHostDiff(t *testing.T) { } func TestAcquireHostSame(t *testing.T) { - c := createConfig() + c := createConfig(&ha_helper.BindUtilsMock{}, options{}) f1 := c.AcquireHost("h1") f2 := c.AcquireHost("h1") if f1 != f2 { @@ -58,3 +71,50 @@ func TestBuildID(t *testing.T) { } } } + +func TestEqual(t *testing.T) { + c1 := createConfig(&ha_helper.BindUtilsMock{}, options{}) + c2 := createConfig(&ha_helper.BindUtilsMock{}, options{}) + if !c1.Equals(c2) { + t.Error("c1 and c2 should be equals (empty)") + } + c1.ConfigDefaultX509Cert("/var/default.pem") + if c1.Equals(c2) { + t.Error("c1 and c2 should not be equals (one default cert)") + } + c2.ConfigDefaultX509Cert("/var/default.pem") + if !c1.Equals(c2) { + t.Error("c1 and c2 should be equals (default cert)") + } + b1 := c1.AcquireBackend("d", "app1", 8080) + c1.AcquireBackend("d", "app2", 8080) + if c1.Equals(c2) { + t.Error("c1 and c2 should not be equals (backends on one side)") + } + c2.AcquireBackend("d", "app2", 8080) + b2 := c2.AcquireBackend("d", "app1", 8080) + if !c1.Equals(c2) { + t.Error("c1 and c2 should be equals (with backends)") + } + h1 := c1.AcquireHost("d") + h1.AddPath(b1, "/") + if c1.Equals(c2) { + t.Error("c1 and c2 should not be equals (hosts on one side)") + } + h2 := c2.AcquireHost("d") + h2.AddPath(b2, "/") + if !c1.Equals(c2) { + t.Error("c1 and c2 should be equals (with hosts)") + } + _, err1 := c1.BuildFrontendGroup() + _, err2 := c2.BuildFrontendGroup() + if err1 != nil { + t.Errorf("error building c1: %v", err1) + } + if err2 != nil { + t.Errorf("error building c2: %v", err2) + } + if !c1.Equals(c2) { + t.Error("c1 and c2 should be equals (after building frontends)") + } +} diff --git a/pkg/haproxy/helper_test/bindutilsmock.go b/pkg/haproxy/helper_test/bindutilsmock.go new file mode 100644 index 000000000..e2414a96a --- /dev/null +++ b/pkg/haproxy/helper_test/bindutilsmock.go @@ -0,0 +1,38 @@ +/* +Copyright 2019 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper_test + +// BindUtilsMock ... +type BindUtilsMock struct { + CertDirs []*CertDir +} + +// CertDir ... +type CertDir struct { + Dir string + Certs []string +} + +// CreateX509CertsDir ... +func (b *BindUtilsMock) CreateX509CertsDir(bindName string, certs []string) (string, error) { + dir := "/var/haproxy/certs/" + bindName + b.CertDirs = append(b.CertDirs, &CertDir{ + Dir: dir, + Certs: certs, + }) + return dir, nil +} diff --git a/pkg/haproxy/instance.go b/pkg/haproxy/instance.go index 2f75d938d..6bd5fde55 100644 --- a/pkg/haproxy/instance.go +++ b/pkg/haproxy/instance.go @@ -22,76 +22,113 @@ import ( "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/dynconfig" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/template" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/types" ) // InstanceOptions ... type InstanceOptions struct { + MaxOldConfigFiles int HAProxyCmd string - ReloadCmd string HAProxyConfigFile string + ReloadCmd string ReloadStrategy string } // Instance ... type Instance interface { - CreateConfig() Config + ParseTemplates() error Config() Config - Templates() *template.Config Update() } // CreateInstance ... -func CreateInstance(logger types.Logger, options InstanceOptions) Instance { - tmpl := &template.Config{ - Logger: logger, - } +func CreateInstance(logger types.Logger, bindUtils hatypes.BindUtils, options InstanceOptions) Instance { dynconf := &dynconfig.Config{ Logger: logger, } return &instance{ - logger: logger, - options: &options, - templates: tmpl, - dynconfig: dynconf, - curConfig: createConfig(), + logger: logger, + bindUtils: bindUtils, + options: &options, + templates: template.CreateConfig(), + mapsTemplate: template.CreateConfig(), + mapsDir: "/etc/haproxy/maps", + dynconfig: dynconf, } } type instance struct { - logger types.Logger - options *InstanceOptions - templates *template.Config - dynconfig *dynconfig.Config - oldConfig Config - curConfig Config -} - -func (i *instance) Templates() *template.Config { - return i.templates + logger types.Logger + bindUtils hatypes.BindUtils + options *InstanceOptions + templates *template.Config + mapsTemplate *template.Config + mapsDir string + dynconfig *dynconfig.Config + oldConfig Config + curConfig Config } -func (i *instance) CreateConfig() Config { - i.releaseConfig() - i.oldConfig = i.curConfig - i.curConfig = createConfig() - return i.curConfig +func (i *instance) ParseTemplates() error { + i.templates.ClearTemplates() + i.mapsTemplate.ClearTemplates() + if err := i.templates.NewTemplate( + "spoe-modsecurity.tmpl", + "/etc/haproxy/modsecurity/spoe-modsecurity.tmpl", + "/etc/haproxy/spoe-modsecurity.conf", + 0, + 1024, + ); err != nil { + return err + } + if err := i.templates.NewTemplate( + "haproxy.tmpl", + "/etc/haproxy/template/haproxy.tmpl", + "/etc/haproxy/haproxy.cfg", + i.options.MaxOldConfigFiles, + 16384, + ); err != nil { + return err + } + err := i.mapsTemplate.NewTemplate( + "map.tmpl", + "/etc/haproxy/maptemplate/map.tmpl", + "", + 0, + 2048, + ) + return err } func (i *instance) Config() Config { + if i.curConfig == nil { + config := createConfig(i.bindUtils, options{ + mapsTemplate: i.mapsTemplate, + mapsDir: i.mapsDir, + }) + i.curConfig = config + } return i.curConfig } func (i *instance) Update() { + if i.curConfig == nil { + i.logger.InfoV(2, "new configuration is empty") + return + } if i.curConfig.Equals(i.oldConfig) { i.logger.InfoV(2, "old and new configurations match, skipping reload") + i.clearConfig() return } - updated := i.dynconfig.Update() - if err := i.templates.Write(i.Config()); err != nil { + if err := i.templates.Write(i.curConfig); err != nil { i.logger.Error("error writing configuration: %v", err) + i.clearConfig() return } + updated := i.dynconfig.Update() + i.clearConfig() if err := i.check(); err != nil { i.logger.Error("error validating config file:\n%v", err) return @@ -104,12 +141,15 @@ func (i *instance) Update() { i.logger.Error("error reloading server:\n%v", err) return } + i.clearConfig() i.logger.Info("HAProxy successfully reloaded") } func (i *instance) check() error { - i.logger.Info("VERIFIED! (skipped)") - return nil + if i.options.HAProxyCmd == "" { + i.logger.Info("(test) check was skipped") + return nil + } out, err := exec.Command(i.options.HAProxyCmd, "-c", "-f", i.options.HAProxyConfigFile).CombinedOutput() if err != nil { return fmt.Errorf(string(out)) @@ -118,8 +158,10 @@ func (i *instance) check() error { } func (i *instance) reload() error { - i.logger.Info("RELOADED! (skipped)") - return nil + if i.options.ReloadCmd == "" { + i.logger.Info("(test) reload was skipped") + return nil + } out, err := exec.Command(i.options.ReloadCmd, i.options.ReloadStrategy, i.options.HAProxyConfigFile).CombinedOutput() if len(out) > 0 { return fmt.Errorf(string(out)) @@ -129,6 +171,8 @@ func (i *instance) reload() error { return nil } -func (i *instance) releaseConfig() { - // TODO +func (i *instance) clearConfig() { + // TODO releaseConfig (old support files, ...) + i.oldConfig = i.curConfig + i.curConfig = nil } diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go new file mode 100644 index 000000000..bb24be3eb --- /dev/null +++ b/pkg/haproxy/instance_test.go @@ -0,0 +1,799 @@ +/* +Copyright 2019 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package haproxy + +import ( + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" + "testing" + + "github.com/kylelemons/godebug/diff" + + ha_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/helper_test" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" + "github.com/jcmoraisjr/haproxy-ingress/pkg/types/helper_test" +) + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * TEMPLATES + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +func TestInstanceEmpty(t *testing.T) { + c := setup(t) + defer c.teardown() + + template := ` +global + daemon + quiet + stats socket %s level admin expose-fd listeners + maxconn 0 + lua-load /usr/local/etc/haproxy/lua/send-response.lua + lua-load /usr/local/etc/haproxy/lua/auth-request.lua + tune.ssl.default-dh-param 0 +defaults + log global + maxconn 0 + option redispatch + option dontlognull + option http-server-close + option http-keep-alive + timeout client %s + timeout connect %s + timeout server %s +backend default_empty_8080 + mode http +backend _error404 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/404.http + http-request deny deny_status 400 +backend _error495 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/495.http + http-request deny deny_status 400 +backend _error496 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/496.http + http-request deny deny_status 400 +frontend _front__http + mode http + bind :80 + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) + use_backend %%[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _error404 +frontend https-front_empty + mode http + bind :443 + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_empty.map,_nomatch) + use_backend %%[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _error404 +` + + c.config.AcquireHost("empty").AddPath(c.config.AcquireBackend("default", "empty", 8080), "/") + c.instance.Update() + c.checkConfigFull(fmt.Sprintf(template, "--", "--", "--", "--")) + + c.checkMap("http-front.map", ` +empty/ default_empty_8080`) + c.checkMap("https-front_empty.map", ` +empty/ default_empty_8080`) + + c.logger.CompareLogging(defaultLogging) +} + +func TestInstanceDefaultHost(t *testing.T) { + c := setup(t) + defer c.teardown() + + c.configGlobal() + def := c.config.AcquireBackend("default", "default-backend", 8080) + def.Endpoints = []*hatypes.Endpoint{endpointS0} + c.config.ConfigDefaultBackend(def) + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.AcquireBackend("d1", "app", 8080) + h = c.config.AcquireHost("*") + h.AddPath(b, "/") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + h.VarNamespace = true + + b = c.config.AcquireBackend("d2", "app", 8080) + h = c.config.AcquireHost("d2.local") + h.AddPath(b, "/app") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + h.VarNamespace = true + + c.instance.Update() + c.checkConfig(` +backend d1_app_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend d2_app_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend _default_backend + mode http + server s0 172.17.0.99:8080 weight 100`, ` +frontend _front__http + mode http + bind :80 + http-request set-var(req.base) base,regsub(:[0-9]+/,/) + redirect scheme https if { var(req.base) -i -m beg d2.local/app } + use_backend d1_app_8080 +frontend https-front_d2.local + mode http + bind :443 ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem + http-request set-var(txn.namespace) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d2.local_k8s_ns.map,-) + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d2.local.map,_nomatch) + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + use_backend d1_app_8080 +`) + + c.checkMap("https-front_d2.local_k8s_ns.map", ` +d2.local/app d2`) + c.checkMap("https-front_d2.local.map", ` +d2.local/app d2_app_8080`) + + c.logger.CompareLogging(defaultLogging) +} + +func TestInstanceSingleFrontendSingleBind(t *testing.T) { + c := setup(t) + defer c.teardown() + + c.configGlobal() + def := c.config.AcquireBackend("default", "default-backend", 8080) + def.Endpoints = []*hatypes.Endpoint{endpointS0} + c.config.ConfigDefaultBackend(def) + c.config.ConfigDefaultX509Cert("/var/haproxy/ssl/certs/default.pem") + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.AcquireBackend("d1", "app", 8080) + h = c.config.AcquireHost("d1.local") + h.AddPath(b, "/") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h.VarNamespace = true + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/d1.pem" + h.TLS.TLSHash = "1" + + b = c.config.AcquireBackend("d2", "app", 8080) + h = c.config.AcquireHost("d2.local") + h.AddPath(b, "/app") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/d2.pem" + h.TLS.TLSHash = "2" + + c.instance.Update() + c.checkConfig(` +backend d1_app_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend d2_app_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend _default_backend + mode http + server s0 172.17.0.99:8080 weight 100`, ` +frontend _front__http + mode http + bind :80 + http-request set-var(req.base) base,regsub(:[0-9]+/,/) + redirect scheme https if { var(req.base) -i -m beg d1.local/ d2.local/app } + default_backend _default_backend +frontend _front_001 + mode http + bind :443 ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem crt /var/haproxy/certs/_public + http-request set-var(txn.namespace) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001_k8s_ns.map,-) + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001.map,_nomatch) + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _default_backend +`) + + c.checkMap("_front_001_k8s_ns.map", ` +d1.local/ d1 +d2.local/app -`) + c.checkMap("_front_001.map", ` +d1.local/ d1_app_8080 +d2.local/app d2_app_8080`) + + c.logger.CompareLogging(defaultLogging) +} + +func TestInstanceSingleFrontendTwoBindsCA(t *testing.T) { + c := setup(t) + defer c.teardown() + + c.configGlobal() + def := c.config.AcquireBackend("default", "default-backend", 8080) + def.Endpoints = []*hatypes.Endpoint{endpointS0} + c.config.ConfigDefaultBackend(def) + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.AcquireBackend("d", "app", 8080) + h = c.config.AcquireHost("d1.local") + h.AddPath(b, "/") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + h.TLS.CAFilename = "/var/haproxy/ssl/ca/d1.local.pem" + h.TLS.CAHash = "1" + h.TLS.CAErrorPage = "http://d1.local/error.html" + + h = c.config.AcquireHost("d2.local") + h.AddPath(b, "/") + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + h.TLS.CAFilename = "/var/haproxy/ssl/ca/d2.local.pem" + h.TLS.CAHash = "2" + + c.instance.Update() + c.checkConfig(` +backend d_app_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend _default_backend + mode http + server s0 172.17.0.99:8080 weight 100`, ` +listen _front__tls + mode tcp + bind :443 + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } + ## _front_001 + use-server _server_d1.local if { req.ssl_sni -i d1.local } + server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 + use-server _server_d2.local if { req.ssl_sni -i d2.local } + server _server_d2.local unix@/var/run/front_d2.local.sock send-proxy-v2 + # TODO default backend +frontend _front__http + mode http + bind :80 + http-request set-var(req.base) base,regsub(:[0-9]+/,/) + redirect scheme https if { var(req.base) -i -m beg d1.local/ d2.local/ } + default_backend _default_backend +frontend _front_001 + mode http + bind unix@/var/run/front_d1.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d1.local.pem verify optional ca-ignore-err all crt-ignore-err all + bind unix@/var/run/front_d2.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d2.local.pem verify optional ca-ignore-err all crt-ignore-err all + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001.map,_nomatch) + acl tls-invalid-crt ssl_c_ca_err gt 0 + acl tls-invalid-crt ssl_c_err gt 0 + acl tls-has-crt ssl_c_used + http-request set-var(req.tls_invalid_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt + redirect location %[var(req.tls_invalid_redir)] code 303 if { var(req.tls_invalid_redir) -m found } !{ var(req.tls_invalid_redir) _internal } + use_backend _error495 if { var(req.tls_invalid_redir) _internal } { ssl_fc_sni -i d1.local d2.local } + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _default_backend +`) + + c.checkMap("_front_001_inv_crt.map", ` +d1.local http://d1.local/error.html`) + c.checkMap("_front_001_no_crt.map", ` +d1.local http://d1.local/error.html`) + c.checkMap("_front_001.map", ` +d1.local/ d_app_8080 +d2.local/ d_app_8080`) + + c.logger.CompareLogging(defaultLogging) +} + +func TestInstanceTwoFrontendsThreeBindsCA(t *testing.T) { + c := setup(t) + defer c.teardown() + + c.configGlobal() + def := c.config.AcquireBackend("default", "default-backend", 8080) + def.Endpoints = []*hatypes.Endpoint{endpointS0} + c.config.ConfigDefaultBackend(def) + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.AcquireBackend("d", "appca", 8080) + h = c.config.AcquireHost("d1.local") + h.AddPath(b, "/") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h.Timeout.Client = "1s" + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + h.TLS.CAFilename = "/var/haproxy/ssl/ca/d1.local.pem" + h.TLS.CAHash = "1" + h.TLS.CAVerifyOptional = true + h.TLS.CAErrorPage = "http://d1.local/error.html" + + h = c.config.AcquireHost("d2.local") + h.AddPath(b, "/") + h.Timeout.Client = "2s" + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + h.TLS.CAFilename = "/var/haproxy/ssl/ca/d2.local.pem" + h.TLS.CAHash = "1" + + b = c.config.AcquireBackend("d", "app", 8080) + h = c.config.AcquireHost("d3.local") + h.AddPath(b, "/") + b.Endpoints = []*hatypes.Endpoint{endpointS21} + h.Timeout.Client = "2s" + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + + h = c.config.AcquireHost("d4.local") + h.AddPath(b, "/") + h.Timeout.Client = "2s" + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + + c.instance.Update() + c.checkConfig(` +backend d_app_8080 + mode http + server s21 172.17.0.121:8080 weight 100 +backend d_appca_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend _default_backend + mode http + server s0 172.17.0.99:8080 weight 100`, ` +listen _front__tls + mode tcp + bind :443 + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } + ## _front_001 + use-server _server_d2.local if { req.ssl_sni -i d2.local } + server _server_d2.local unix@/var/run/front_d2.local.sock send-proxy-v2 + use-server _server__socket001 if { req.ssl_sni -i d3.local d4.local } + server _server__socket001 unix@/var/run/front__socket001.sock send-proxy-v2 + ## https-front_d1.local + use-server _server_d1.local if { req.ssl_sni -i d1.local } + server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 + # TODO default backend +frontend _front__http + mode http + bind :80 + http-request set-var(req.base) base,regsub(:[0-9]+/,/) + http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) + redirect scheme https if { var(req.base) -i -m beg d1.local/ d2.local/ } + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _default_backend +frontend _front_001 + mode http + bind unix@/var/run/front_d2.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d2.local.pem verify optional ca-ignore-err all crt-ignore-err all + bind unix@/var/run/front__socket001.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/certs/_socket001 + timeout client 2s + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001.map,_nomatch) + acl tls-invalid-crt ssl_c_ca_err gt 0 + acl tls-invalid-crt ssl_c_err gt 0 + acl tls-has-crt ssl_c_used + http-request set-var(req.tls_invalid_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt + use_backend _error495 if { var(req.tls_invalid_redir) _internal } { ssl_fc_sni -i d2.local } + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _default_backend +frontend https-front_d1.local + mode http + bind unix@/var/run/front_d1.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d1.local.pem verify optional ca-ignore-err all crt-ignore-err all + timeout client 1s + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d1.local.map,_nomatch) + acl tls-invalid-crt ssl_c_ca_err gt 0 + acl tls-invalid-crt ssl_c_err gt 0 + acl tls-has-crt ssl_c_used + http-request set-var(req.tls_invalid_redir) ssl_fc_sni,map(/etc/haproxy/maps/https-front_d1.local_inv_crt.map,_internal) if tls-invalid-crt + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/https-front_d1.local_no_crt.map,_internal) if !tls-has-crt + redirect location %[var(req.tls_invalid_redir)] code 303 if { var(req.tls_invalid_redir) -m found } !{ var(req.tls_invalid_redir) _internal } + redirect location %[var(req.tls_nocrt_redir)] code 303 if { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } + use_backend _error495 if { var(req.tls_invalid_redir) _internal } { ssl_fc_sni -i d1.local } + use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i d1.local } + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _default_backend +`) + + c.checkMap("http-front.map", ` +d3.local/ d_app_8080 +d4.local/ d_app_8080`) + c.checkMap("_front_001_inv_crt.map", ` +`) + c.checkMap("_front_001_no_crt.map", ` +`) + c.checkMap("_front_001.map", ` +d2.local/ d_appca_8080 +d3.local/ d_app_8080 +d4.local/ d_app_8080`) + c.checkMap("https-front_d1.local_inv_crt.map", ` +d1.local http://d1.local/error.html`) + c.checkMap("https-front_d1.local_no_crt.map", ` +d1.local http://d1.local/error.html`) + c.checkMap("https-front_d1.local.map", ` +d1.local/ d_appca_8080`) + + c.logger.CompareLogging(defaultLogging) +} + +func TestInstanceSomePaths(t *testing.T) { + c := setup(t) + defer c.teardown() + + c.configGlobal() + def := c.config.AcquireBackend("default", "default-backend", 8080) + def.Endpoints = []*hatypes.Endpoint{endpointS0} + c.config.ConfigDefaultBackend(def) + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.AcquireBackend("d", "app0", 8080) + h = c.config.AcquireHost("d.local") + h.AddPath(b, "/") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + + b = c.config.AcquireBackend("d", "app1", 8080) + h.AddPath(b, "/app") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS1} + + b = c.config.AcquireBackend("d", "app2", 8080) + h.AddPath(b, "/app/sub") + b.Endpoints = []*hatypes.Endpoint{endpointS21, endpointS22} + + b = c.config.AcquireBackend("d", "app3", 8080) + h.AddPath(b, "/sub") + b.Endpoints = []*hatypes.Endpoint{endpointS31, endpointS32, endpointS33} + + c.instance.Update() + c.checkConfig(` +backend d_app0_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend d_app1_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend d_app2_8080 + mode http + server s21 172.17.0.121:8080 weight 100 + server s22 172.17.0.122:8080 weight 100 +backend d_app3_8080 + mode http + server s31 172.17.0.131:8080 weight 100 + server s32 172.17.0.132:8080 weight 100 + server s33 172.17.0.133:8080 weight 100 +backend _default_backend + mode http + server s0 172.17.0.99:8080 weight 100`, ` +frontend _front__http + mode http + bind :80 + http-request set-var(req.base) base,regsub(:[0-9]+/,/) + http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) + redirect scheme https if { var(req.base) -i -m beg d.local/app d.local/ } + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _default_backend +frontend https-front_d.local + mode http + bind :443 ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d.local.map,_nomatch) + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _default_backend +`) + + c.checkMap("https-front_d.local.map", ` +d.local/sub d_app3_8080 +d.local/app/sub d_app2_8080 +d.local/app d_app1_8080 +d.local/ d_app0_8080`) + + c.logger.CompareLogging(defaultLogging) +} + +func TestSSLPassthrough(t *testing.T) { + c := setup(t) + defer c.teardown() + + c.configGlobal() + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.AcquireBackend("d2", "app", 8080) + h = c.config.AcquireHost("d2.local") + h.AddPath(b, "/") + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS31} + h.SSLPassthrough = true + + b = c.config.AcquireBackend("d3", "app-ssl", 8443) + h = c.config.AcquireHost("d3.local") + h.AddPath(b, "/") + b.Endpoints = []*hatypes.Endpoint{endpointS41s} + h.SSLPassthrough = true + + b = c.config.AcquireBackend("d3", "app-http", 8080) + b.SSLRedirect = true + b.Endpoints = []*hatypes.Endpoint{endpointS41h} + h.HTTPPassthroughBackend = b + + c.instance.Update() + c.checkConfig(` +backend d2_app_8080 + mode http + server s31 172.17.0.131:8080 weight 100 +backend d3_app-http_8080 + mode http + server s41h 172.17.0.141:8080 weight 100 +backend d3_app-ssl_8443 + mode http + server s41s 172.17.0.141:8443 weight 100 +backend _error404 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/404.http + http-request deny deny_status 400`, ` +listen _front__tls + mode tcp + bind :443 + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } + ## ssl-passthrough + tcp-request content set-var(req.backend) req.ssl_sni,lower,map(/etc/haproxy/maps/sslpassthrough.map,_nomatch) + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + # TODO default backend +frontend _front__http + mode http + bind :80 + http-request set-var(req.base) base,regsub(:[0-9]+/,/) + http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) + redirect scheme https if { var(req.base) -i -m beg d2.local/ } + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + default_backend _error404`) + + c.checkMap("sslpassthrough.map", ` +d2.local d2_app_8080 +d3.local d3_app-ssl_8443`) + c.checkMap("http-front.map", ` +d3.local/ d3_app-http_8080`) + + c.logger.CompareLogging(defaultLogging) +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * + * BUILDERS + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +type testConfig struct { + t *testing.T + logger *helper_test.LoggerMock + instance Instance + config Config + tempdir string + configfile string +} + +func setup(t *testing.T) *testConfig { + logger := &helper_test.LoggerMock{T: t} + tempdir, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("error creating tempdir: %v", err) + } + configfile := tempdir + "/haproxy.cfg" + instance := CreateInstance(logger, &ha_helper.BindUtilsMock{}, InstanceOptions{ + HAProxyConfigFile: configfile, + }).(*instance) + if err := instance.templates.NewTemplate( + "haproxy.tmpl", + "../../rootfs/etc/haproxy/template/haproxy.tmpl", + configfile, + 0, + 2048, + ); err != nil { + t.Errorf("error parsing haproxy.tmpl: %v", err) + } + if err := instance.mapsTemplate.NewTemplate( + "map.tmpl", + "../../rootfs/etc/haproxy/maptemplate/map.tmpl", + "", + 0, + 2048, + ); err != nil { + t.Errorf("error parsing map.tmpl: %v", err) + } + config := createConfig(&ha_helper.BindUtilsMock{}, options{ + mapsTemplate: instance.mapsTemplate, + mapsDir: tempdir, + }) + instance.curConfig = config + return &testConfig{ + t: t, + logger: logger, + instance: instance, + config: config, + tempdir: tempdir, + configfile: configfile, + } +} + +func (c *testConfig) teardown() { + c.logger.CompareLogging("") + if err := os.RemoveAll(c.tempdir); err != nil { + c.t.Errorf("error removing tempdir: %v", err) + } +} + +func (c *testConfig) configGlobal() { + global := c.config.Global() + global.MaxConn = 2000 + global.SSL.Ciphers = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256" + global.SSL.DHParam.Filename = "/var/haproxy/tls/dhparam.pem" + global.SSL.Options = "no-sslv3" + global.StatsSocket = "/var/run/haproxy.sock" + global.Timeout.Client = "50s" + global.Timeout.ClientFin = "50s" + global.Timeout.Connect = "5s" + global.Timeout.HTTPRequest = "5s" + global.Timeout.KeepAlive = "1m" + global.Timeout.Queue = "5s" + global.Timeout.Server = "50s" + global.Timeout.ServerFin = "50s" + global.Timeout.Stop = "15m" + global.Timeout.Tunnel = "1h" +} + +var globalConfig = ` +global + daemon + quiet + stats socket /var/run/haproxy.sock level admin expose-fd listeners + maxconn 2000 + hard-stop-after 15m + lua-load /usr/local/etc/haproxy/lua/send-response.lua + lua-load /usr/local/etc/haproxy/lua/auth-request.lua + ssl-dh-param-file /var/haproxy/tls/dhparam.pem + ssl-default-bind-ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256 + ssl-default-bind-options no-sslv3 +defaults + log global + maxconn 2000 + option redispatch + option dontlognull + option http-server-close + option http-keep-alive + timeout client 50s + timeout client-fin 50s + timeout connect 5s + timeout http-keep-alive 1m + timeout http-request 5s + timeout queue 5s + timeout server 50s + timeout server-fin 50s + timeout tunnel 1h` + +var errorPages = ` +backend _error495 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/495.http + http-request deny deny_status 400 +backend _error496 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/496.http + http-request deny deny_status 400` + +var endpointS0 = &hatypes.Endpoint{ + Name: "s0", + IP: "172.17.0.99", + Port: 8080, + Weight: 100, +} +var endpointS1 = &hatypes.Endpoint{ + Name: "s1", + IP: "172.17.0.11", + Port: 8080, + Weight: 100, +} +var endpointS21 = &hatypes.Endpoint{ + Name: "s21", + IP: "172.17.0.121", + Port: 8080, + Weight: 100, +} +var endpointS22 = &hatypes.Endpoint{ + Name: "s22", + IP: "172.17.0.122", + Port: 8080, + Weight: 100, +} +var endpointS31 = &hatypes.Endpoint{ + Name: "s31", + IP: "172.17.0.131", + Port: 8080, + Weight: 100, +} +var endpointS32 = &hatypes.Endpoint{ + Name: "s32", + IP: "172.17.0.132", + Port: 8080, + Weight: 100, +} +var endpointS33 = &hatypes.Endpoint{ + Name: "s33", + IP: "172.17.0.133", + Port: 8080, + Weight: 100, +} +var endpointS41s = &hatypes.Endpoint{ + Name: "s41s", + IP: "172.17.0.141", + Port: 8443, + Weight: 100, +} +var endpointS41h = &hatypes.Endpoint{ + Name: "s41h", + IP: "172.17.0.141", + Port: 8080, + Weight: 100, +} + +var defaultLogging = ` +INFO (test) check was skipped +INFO (test) reload was skipped +INFO HAProxy successfully reloaded` + +func (c *testConfig) checkConfig(backend, frontend string) { + c.checkConfigFull(globalConfig + backend + errorPages + frontend) +} + +func (c *testConfig) checkConfigFull(expected string) { + actual := strings.Replace(c.readConfig(c.configfile), c.tempdir, "/etc/haproxy/maps", -1) + c.compareText("haproxy.cfg", actual, expected) +} + +func (c *testConfig) checkMap(mapName, expected string) { + actual := c.readConfig(c.tempdir + "/" + mapName) + c.compareText(mapName, actual, expected) +} + +var replaceComments = regexp.MustCompile(`(?m)^[ \t]{0,2}(#.*)?[\r\n]+`) + +func (c *testConfig) readConfig(fileName string) string { + config, err := ioutil.ReadFile(fileName) + if err != nil { + c.t.Errorf("error reading config file: %v", err) + return "" + } + configStr := replaceComments.ReplaceAllString(string(config), ``) + return configStr +} + +func (c *testConfig) compareText(name, actual, expected string) { + txtActual := "\n" + strings.Trim(actual, "\n") + txtExpected := "\n" + strings.Trim(expected, "\n") + if txtActual != txtExpected { + c.t.Error("\ndiff of " + name + ":" + diff.Diff(txtExpected, txtActual)) + } +} diff --git a/pkg/haproxy/template/template.go b/pkg/haproxy/template/template.go index 0951f311d..8745e1fbd 100644 --- a/pkg/haproxy/template/template.go +++ b/pkg/haproxy/template/template.go @@ -22,22 +22,28 @@ import ( "io/ioutil" "os" gotemplate "text/template" - - "github.com/jcmoraisjr/haproxy-ingress/pkg/types" ) +// CreateConfig ... +func CreateConfig() *Config { + return &Config{} +} + // Config ... type Config struct { - Logger types.Logger templates []*template } +// ClearTemplates ... +func (c *Config) ClearTemplates() { + c.templates = nil +} + // NewTemplate ... -func (c *Config) NewTemplate(name, file, output string, rotate, startingBufferSize int) { +func (c *Config) NewTemplate(name, file, output string, rotate, startingBufferSize int) error { tmpl, err := gotemplate.New(name).Funcs(funcMap).ParseFiles(file) if err != nil { - c.Logger.Fatal("cannot read template file: %v", err) - return // unit tests need this + return fmt.Errorf("cannot read template file: %v", err) } c.templates = append(c.templates, &template{ tmpl: tmpl, @@ -45,10 +51,16 @@ func (c *Config) NewTemplate(name, file, output string, rotate, startingBufferSi rotate: rotate, rawConfig: bytes.NewBuffer(make([]byte, 0, startingBufferSize)), }) + return nil } // Write ... func (c *Config) Write(data interface{}) error { + return c.WriteOutput(data, "") +} + +// WriteOutput ... +func (c *Config) WriteOutput(data interface{}, output string) error { for _, t := range c.templates { t.rawConfig.Reset() if err := t.tmpl.Execute(t.rawConfig, data); err != nil { @@ -56,7 +68,7 @@ func (c *Config) Write(data interface{}) error { } } for _, t := range c.templates { - if err := t.writeToDisk(); err != nil { + if err := t.writeToDisk(output); err != nil { return err } } @@ -71,21 +83,27 @@ type template struct { configFiles []string } -func (t *template) writeToDisk() error { +func (t *template) writeToDisk(output string) error { + if output == "" { + output = t.output + } + if output == "" { + return fmt.Errorf("output file is empty, configure on NewTemplate() or use WriteOutput()") + } if t.rotate > 0 { // Include timestamp in rotated config file names to aid troubleshooting. // When using a single, ever-changing config file it was difficult // to know what config was loaded by any given haproxy process // // rename current config file, if exists - if f, err := os.Stat(t.output); f != nil { - rotateTo := t.output + "." + f.ModTime().Format("20060102-150405.000") - if err := os.Rename(t.output, rotateTo); err != nil { - return fmt.Errorf("cannot rotate %s: %v", t.output, err) + if f, err := os.Stat(output); f != nil { + rotateTo := output + "." + f.ModTime().Format("20060102-150405.000") + if err := os.Rename(output, rotateTo); err != nil { + return fmt.Errorf("cannot rotate %s: %v", output, err) } t.configFiles = append(t.configFiles, rotateTo) } else if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("cannot rotate %s: %v", t.output, err) + return fmt.Errorf("cannot rotate %s: %v", output, err) } // remove old config files for len(t.configFiles) > t.rotate { @@ -96,8 +114,8 @@ func (t *template) writeToDisk() error { t.configFiles = t.configFiles[1:] } } - if err := ioutil.WriteFile(t.output, t.rawConfig.Bytes(), 0644); err != nil { - return fmt.Errorf("cannot write %s: %v", t.output, err) + if err := ioutil.WriteFile(output, t.rawConfig.Bytes(), 0644); err != nil { + return fmt.Errorf("cannot write %s: %v", output, err) } return nil } diff --git a/pkg/haproxy/template/template_test.go b/pkg/haproxy/template/template_test.go index b802d566a..5ae6baa82 100644 --- a/pkg/haproxy/template/template_test.go +++ b/pkg/haproxy/template/template_test.go @@ -35,11 +35,21 @@ type testConfig struct { tempdirOutput string } +func TestEmpty(t *testing.T) { + c := setup(t) + defer c.teardown() + if err := c.templateConfig.Write(nil); err != nil { + t.Errorf("error writing an empty template: %v", err) + } +} + func TestNewTemplateFileNotFound(t *testing.T) { c := setup(t) defer c.teardown() - c.templateConfig.NewTemplate("h.cfg", "/file", "/tmp/out", 0, 1024) - c.logger.CompareLogging("FATAL cannot read template file: open /file: no such file or directory") + err := c.templateConfig.NewTemplate("h.cfg", "/file", "/tmp/out", 0, 1024) + if err == nil { + t.Errorf("expected error") + } } func TestWrite(t *testing.T) { @@ -259,7 +269,9 @@ func (c *testConfig) newTemplate(content string, rotate int) { if err := ioutil.WriteFile(templatePath, []byte(content), 0644); err != nil { c.t.Errorf("error writing template file: %v", err) } - c.templateConfig.NewTemplate(templateFileName, templatePath, outputPath, rotate, 1024) + if err := c.templateConfig.NewTemplate(templateFileName, templatePath, outputPath, rotate, 1024); err != nil { + c.t.Errorf("error parsing %s: %v", templateFileName, err) + } } func (c *testConfig) outputs(index int) []string { @@ -282,13 +294,11 @@ func setup(t *testing.T) *testConfig { t.Errorf("error creating tempdir: %v", err) } return &testConfig{ - t: t, - logger: logger, - templateConfig: &Config{ - Logger: logger, - }, - tempdir: tempdir, - tempdirOutput: tempdir, + t: t, + logger: logger, + templateConfig: CreateConfig(), + tempdir: tempdir, + tempdirOutput: tempdir, } } diff --git a/pkg/haproxy/types/backend.go b/pkg/haproxy/types/backend.go index 23a6c01cc..6c43850a5 100644 --- a/pkg/haproxy/types/backend.go +++ b/pkg/haproxy/types/backend.go @@ -22,16 +22,17 @@ import ( ) // NewEndpoint ... -func (b *Backend) NewEndpoint(ip string, port int, target string) *Endpoint { +func (b *Backend) NewEndpoint(ip string, port int, targetRef string) *Endpoint { endpoint := &Endpoint{ - IP: ip, - Port: port, - Target: target, - Weight: 1, + Name: fmt.Sprintf("%s:%d", ip, port), + IP: ip, + Port: port, + TargetRef: targetRef, + Weight: 1, } b.Endpoints = append(b.Endpoints, endpoint) sort.Slice(b.Endpoints, func(i, j int) bool { - return b.Endpoints[i].IP < b.Endpoints[j].IP + return b.Endpoints[i].Name < b.Endpoints[j].Name }) return endpoint } diff --git a/pkg/haproxy/types/frontend.go b/pkg/haproxy/types/frontend.go new file mode 100644 index 000000000..c4887a8b5 --- /dev/null +++ b/pkg/haproxy/types/frontend.go @@ -0,0 +1,176 @@ +/* +Copyright 2019 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "fmt" + "reflect" + "sort" +) + +// HasTCPProxy ... +func (fg *FrontendGroup) HasTCPProxy() bool { + // short-circuit saves: + // len(fg.Frontend) may be zero only if fg.HasSSLPassthrough is true + return fg.HasSSLPassthrough || len(fg.Frontends) > 1 || len(fg.Frontends[0].Binds) > 1 +} + +// String ... +func (f *Frontend) String() string { + return fmt.Sprintf("%+v", *f) +} + +// HasTLSAuth ... +func (f *Frontend) HasTLSAuth() bool { + for _, host := range f.Hosts { + if host.HasTLSAuth() { + return true + } + } + return false +} + +// HasInvalidErrorPage ... +func (f *Frontend) HasInvalidErrorPage() bool { + for _, host := range f.Hosts { + if host.TLS.CAErrorPage != "" { + return true + } + } + return false +} + +// HasNoCrtErrorPage ... +func (f *Frontend) HasNoCrtErrorPage() bool { + // Use currently the same attribute + return f.HasInvalidErrorPage() +} + +// HasTLSOptional ... +func (f *Frontend) HasTLSOptional() bool { + for _, host := range f.Hosts { + if host.TLS.CAVerifyOptional { + return true + } + } + return false +} + +// HasVarNamespace ... +func (f *Frontend) HasVarNamespace() bool { + for _, host := range f.Hosts { + if host.VarNamespace { + return true + } + } + return false +} + +// BuildRawFrontends ... +func BuildRawFrontends(hosts []*Host) (frontends []*Frontend, sslpassthrough []*Host) { + if len(hosts) == 0 { + return nil, nil + } + // creating frontends and ssl-passthrough hosts + for _, host := range hosts { + if host.SSLPassthrough { + // ssl-passthrough does not use a frontend + sslpassthrough = append(sslpassthrough, host) + continue + } + frontend := findMatchingFrontend(frontends, host) + if frontend == nil { + frontend = newFrontend(host) + frontends = append(frontends, frontend) + } + frontend.Hosts = append(frontend.Hosts, host) + } + // creating binds + for _, frontend := range frontends { + var binds []*BindConfig + for _, host := range frontend.Hosts { + bind := findMatchingBind(binds, host) + if bind == nil { + bind = newFrontendBind(host) + binds = append(binds, bind) + } + bind.Hosts = append(bind.Hosts, host) + } + frontend.Binds = binds + } + // naming frontends + var i int + for _, frontend := range frontends { + if len(frontend.Hosts) == 1 { + frontend.Name = "https-front_" + frontend.Hosts[0].Hostname + } else { + i++ + frontend.Name = fmt.Sprintf("_front_%03d", i) + } + } + // sorting frontends + sort.Slice(frontends, func(i, j int) bool { + return frontends[i].Name < frontends[j].Name + }) + return frontends, sslpassthrough +} + +func findMatchingFrontend(frontends []*Frontend, host *Host) *Frontend { + for _, frontend := range frontends { + if frontend.match(host) { + return frontend + } + } + return nil +} + +func findMatchingBind(binds []*BindConfig, host *Host) *BindConfig { + for _, bind := range binds { + if bind.match(host) { + return bind + } + } + return nil +} + +// newFrontend and Frontend.Match should always sinchronize its attributes +func newFrontend(host *Host) *Frontend { + return &Frontend{ + Timeout: host.Timeout, + } +} + +// newFrontendBind and BindConfig.Match should always sinchronize its attributes +func newFrontendBind(host *Host) *BindConfig { + return &BindConfig{ + TLS: BindTLSConfig{ + CAFilename: host.TLS.CAFilename, + CAHash: host.TLS.CAHash, + }, + } +} + +func (f *Frontend) match(host *Host) bool { + if len(f.Hosts) == 0 { + return true + } + return reflect.DeepEqual(f.Timeout, host.Timeout) +} + +func (b *BindConfig) match(host *Host) bool { + return b.TLS.CAHash == host.TLS.CAHash +} diff --git a/pkg/haproxy/types/frontend_test.go b/pkg/haproxy/types/frontend_test.go new file mode 100644 index 000000000..cd88d713f --- /dev/null +++ b/pkg/haproxy/types/frontend_test.go @@ -0,0 +1,167 @@ +/* +Copyright 2019 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "testing" + + "github.com/kylelemons/godebug/diff" + yaml "gopkg.in/yaml.v2" +) + +func TestBuildFrontendEmpty(t *testing.T) { + frontends, _ := BuildRawFrontends([]*Host{}) + if len(frontends) > 0 { + t.Errorf("expected len(frontends) == 0, but was %d", len(frontends)) + } +} + +func TestBuildFrontend(t *testing.T) { + timeout10 := HostTimeoutConfig{Client: "10s"} + timeout20 := HostTimeoutConfig{Client: "20s"} + ca1 := HostTLSConfig{CAHash: "1"} + ca2 := HostTLSConfig{CAHash: "2"} + h10_1 := &Host{Hostname: "h1.local", Timeout: timeout10} + h10_2 := &Host{Hostname: "h2.local", Timeout: timeout10} + h20_1 := &Host{Hostname: "h3.local", Timeout: timeout20} + h10CA1_1 := &Host{Hostname: "h4.local", Timeout: timeout10, TLS: ca1} + h10CA2_1 := &Host{Hostname: "h5.local", Timeout: timeout10, TLS: ca2} + h10CA2_2 := &Host{Hostname: "h6.local", Timeout: timeout10, TLS: ca2} + testCases := []struct { + hosts []*Host + expected []*Frontend + }{ + // 0 + { + hosts: []*Host{h10_1, h10_2}, + expected: []*Frontend{ + { + Name: "_front_001", + Timeout: timeout10, + Hosts: []*Host{h10_1, h10_2}, + Binds: []*BindConfig{ + &BindConfig{ + Hosts: []*Host{h10_1, h10_2}, + }, + }, + }, + }, + }, + // 1 + { + hosts: []*Host{h10_1, h20_1, h10_2}, + expected: []*Frontend{ + { + Name: "_front_001", + Timeout: timeout10, + Hosts: []*Host{h10_1, h10_2}, + Binds: []*BindConfig{ + &BindConfig{ + Hosts: []*Host{h10_1, h10_2}, + }, + }, + }, + { + Name: "https-front_h3.local", + Timeout: timeout20, + Hosts: []*Host{h20_1}, + Binds: []*BindConfig{ + &BindConfig{ + Hosts: []*Host{h20_1}, + }, + }, + }, + }, + }, + // 2 + { + hosts: []*Host{h10CA1_1, h10CA2_1, h10CA2_2}, + expected: []*Frontend{ + { + Name: "_front_001", + Timeout: timeout10, + Hosts: []*Host{h10CA1_1, h10CA2_1, h10CA2_2}, + Binds: []*BindConfig{ + &BindConfig{ + Hosts: []*Host{h10CA1_1}, + TLS: BindTLSConfig{CAHash: "1"}, + }, + &BindConfig{ + Hosts: []*Host{h10CA2_1, h10CA2_2}, + TLS: BindTLSConfig{CAHash: "2"}, + }, + }, + }, + }, + }, + // 3 + { + hosts: []*Host{h10_1, h10_2, h10CA2_1, h10CA2_2}, + expected: []*Frontend{ + { + Name: "_front_001", + Timeout: timeout10, + Hosts: []*Host{h10_1, h10_2, h10CA2_1, h10CA2_2}, + Binds: []*BindConfig{ + &BindConfig{ + Hosts: []*Host{h10_1, h10_2}, + }, + &BindConfig{ + Hosts: []*Host{h10CA2_1, h10CA2_2}, + TLS: BindTLSConfig{CAHash: "2"}, + }, + }, + }, + }, + }, + } + for i, test := range testCases { + frontends, _ := BuildRawFrontends(test.hosts) + actualRaw, _ := yaml.Marshal(frontends) + expectedRaw, _ := yaml.Marshal(test.expected) + actual := string(actualRaw) + expected := string(expectedRaw) + if actual != expected { + t.Errorf("frontend '%d' actual and expected differs:\n%v", i, diff.Diff(actual, expected)) + } + } +} + +func TestBuildSSLPassthrough(t *testing.T) { + h1 := &Host{Hostname: "h1.local"} + h2 := &Host{Hostname: "h2.local", SSLPassthrough: true} + testCases := []struct { + hosts []*Host + expected []*Host + }{ + // 0 + { + hosts: []*Host{h1, h2}, + expected: []*Host{h2}, + }, + } + for i, test := range testCases { + _, sslpassthrough := BuildRawFrontends(test.hosts) + actualRaw, _ := yaml.Marshal(sslpassthrough) + expectedRaw, _ := yaml.Marshal(test.expected) + actual := string(actualRaw) + expected := string(expectedRaw) + if actual != expected { + t.Errorf("sslpassthrough '%d' actual and expected differs:\n%v", i, diff.Diff(actual, expected)) + } + } +} diff --git a/pkg/haproxy/types/host.go b/pkg/haproxy/types/host.go index a70646683..6f984f253 100644 --- a/pkg/haproxy/types/host.go +++ b/pkg/haproxy/types/host.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "fmt" "sort" ) @@ -34,10 +35,20 @@ func (h *Host) FindPath(path string) *HostPath { func (h *Host) AddPath(backend *Backend, path string) { h.Paths = append(h.Paths, &HostPath{ Path: path, - Backend: *backend, + Backend: backend, BackendID: backend.ID, }) sort.Slice(h.Paths, func(i, j int) bool { return h.Paths[i].Path > h.Paths[j].Path }) } + +// HasTLSAuth ... +func (h *Host) HasTLSAuth() bool { + return h.TLS.CAHash != "" +} + +// String ... +func (h *Host) String() string { + return fmt.Sprintf("%+v", *h) +} diff --git a/pkg/haproxy/types/interfaces.go b/pkg/haproxy/types/interfaces.go new file mode 100644 index 000000000..d3e7008c3 --- /dev/null +++ b/pkg/haproxy/types/interfaces.go @@ -0,0 +1,22 @@ +/* +Copyright 2019 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +// BindUtils ... +type BindUtils interface { + CreateX509CertsDir(bindName string, certs []string) (string, error) +} diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index e13df230f..b430bf45f 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -83,6 +83,48 @@ type ModSecurityTimeoutConfig struct { Processing string } +// FrontendGroup ... +type FrontendGroup struct { + Frontends []*Frontend + HasHTTPHost bool + HasRedirectHTTPS bool + HasSSLPassthrough bool + HTTPFrontsMap string + SSLPassthroughMap string +} + +// Frontend ... +type Frontend struct { + Name string + Binds []*BindConfig + Hosts []*Host + // + BackendsMap string + ConvertLowercase bool + Timeout HostTimeoutConfig + TLSInvalidCrtErrorPagesMap string + TLSNoCrtErrorPagesMap string + VarNamespaceMap string +} + +// BindConfig ... +type BindConfig struct { + Name string + Socket string + Hosts []*Host + // + AcceptProxy bool + TLS BindTLSConfig +} + +// BindTLSConfig ... +type BindTLSConfig struct { + CAFilename string + CAHash string + TLSCert string + TLSCertDir string +} + // Host ... // // Wildcard `*` hostname is a catch all and will be used if no other hostname, @@ -90,14 +132,16 @@ type ModSecurityTimeoutConfig struct { // the default backend will be used. If the default backend is empty, // a default 404 page generated by HAProxy will be used. type Host struct { - Hostname string - Paths []*HostPath - RootRedirect string + Hostname string + Paths []*HostPath + // Alias HostAliasConfig HTTPPassthroughBackend *Backend + RootRedirect string SSLPassthrough bool Timeout HostTimeoutConfig TLS HostTLSConfig + VarNamespace bool } // HostPath ... @@ -108,7 +152,7 @@ type Host struct { // empty, a default 404 page generated by HAProxy will be used. type HostPath struct { Path string - Backend Backend + Backend *Backend BackendID string } @@ -126,36 +170,73 @@ type HostTimeoutConfig struct { // HostTLSConfig ... type HostTLSConfig struct { - TLSFilename string - TLSFileSHA256Sum string - CAFilename string - CAFileSHA256Sum string AddCertHeader bool - ErrorPage string + CAErrorPage string + CAFilename string + CAHash string + CAVerifyOptional bool + TLSFilename string + TLSHash string } // Backend ... type Backend struct { - ID string - Namespace string - Name string - Port int - Endpoints []*Endpoint - BalanceAlgorithm string - Cookie Cookie - MaxconnServer int - ModeTCP bool - ProxyBodySize string - SSLRedirect bool - HTTPRequests []*HTTPRequest + ID string + Namespace string + Name string + Port int + Endpoints []*Endpoint + // + AgentCheck AgentCheck + BalanceAlgorithm string + Cookie Cookie + CustomConfig []string + HealthCheck HealthCheck + HTTPRequests []*HTTPRequest + MaxConnServer int + MaxQueueServer int + ModeTCP bool + ProxyBodySize string + SendProxyProtocol string + SSL SSLBackendConfig + SSLRedirect bool + Timeout BackendTimeoutConfig } // Endpoint ... type Endpoint struct { - IP string - Port int - Weight int - Target string + Disabled bool + IP string + Name string + Port int + TargetRef string + Weight int +} + +// AgentCheck ... +type AgentCheck struct { + Addr string + Interval string + Port string + Send string +} + +// HealthCheck ... +type HealthCheck struct { + Addr string + FallCount string + Interval string + Port string + RiseCount string +} + +// SSLBackendConfig ... +type SSLBackendConfig struct { + IsSecure bool + CertFilename string + CertHash string + CAFilename string + CAHash string } // BackendTimeoutConfig ... diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index 0d4f6ca02..519a36b6d 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -20,7 +20,7 @@ ARG DUMB_INIT_SHA256=81231da1cd074fdc81af62789fead8641ef3f24b6b07366a1c34e5b059f RUN wget -O/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64\ && echo "$DUMB_INIT_SHA256 /dumb-init" | sha256sum -c -\ && chmod +x /dumb-init \ - && mkdir /ingress-controller + && mkdir -p /ingress-controller /etc/haproxy/maps COPY . / diff --git a/rootfs/etc/haproxy/maptemplate/map.tmpl b/rootfs/etc/haproxy/maptemplate/map.tmpl new file mode 100644 index 000000000..f2ad043ce --- /dev/null +++ b/rootfs/etc/haproxy/maptemplate/map.tmpl @@ -0,0 +1,11 @@ + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# # HAProxy Ingress Controller +# # -------------------------- +# # This file is automatically updated, do not edit +# # +# +{{- range . }} +{{ .Key }} {{ .Value }} +{{- end }} diff --git a/rootfs/etc/haproxy/modsecurity/spoe-modsecurity.tmpl b/rootfs/etc/haproxy/modsecurity/spoe-modsecurity.tmpl new file mode 100644 index 000000000..08a1f98e4 --- /dev/null +++ b/rootfs/etc/haproxy/modsecurity/spoe-modsecurity.tmpl @@ -0,0 +1,20 @@ + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# # HAProxy Ingress Controller +# # -------------------------- +# # This file is automatically updated, do not edit +# # +# +{{- $modsec := .Global.ModSecurity -}} +[modsecurity] +spoe-agent modsecurity-agent + messages check-request + option var-prefix modsec + timeout hello {{ $modsec.Timeout.Hello }} + timeout idle {{ $modsec.Timeout.Idle }} + timeout processing {{ $modsec.Timeout.Processing }} + use-backend spoe-modsecurity +spoe-message check-request + args unique-id method path query req.ver req.hdrs_bin req.body_size req.body + event on-frontend-http-request diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl new file mode 100644 index 000000000..530f4e5ea --- /dev/null +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -0,0 +1,401 @@ + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# # HAProxy Ingress Controller +# # -------------------------- +# # This file is automatically updated, do not edit +# # +# +{{- $cfg := . }} +{{- $global := $cfg.Global }} +global + daemon + quiet +{{- if gt $global.Procs.Nbproc 1 }} + nbproc {{ $global.Procs.Nbproc }} +{{- end }} +{{- if gt $global.Procs.Nbthread 1 }} + nbthread {{ $global.Procs.Nbthread }} +{{- end }} +{{- if $global.Procs.CPUMap }} + cpu-map {{ $global.Procs.CPUMap }} +{{- end }} + stats socket {{ default "--" $global.StatsSocket }} level admin expose-fd listeners + {{- if gt $global.Procs.Nbproc 1 }} process 1{{ end }} +{{- if $global.LoadServerState }} + server-state-file state-global + server-state-base /var/lib/haproxy/ +{{- end }} + maxconn {{ $global.MaxConn }} +{{- if $global.Timeout.Stop }} + hard-stop-after {{ $global.Timeout.Stop }} +{{- end }} +{{- if $global.Syslog.Endpoint }} + log {{ $global.Syslog.Endpoint }} format {{ $global.Syslog.Format }} local0 + log-tag {{ $global.Syslog.Tag }} +{{- end }} + lua-load /usr/local/etc/haproxy/lua/send-response.lua + lua-load /usr/local/etc/haproxy/lua/auth-request.lua +{{- if $global.SSL.DHParam.Filename }} + ssl-dh-param-file {{ $global.SSL.DHParam.Filename }} +{{- else }} + tune.ssl.default-dh-param {{ $global.SSL.DHParam.DefaultMaxSize }} +{{- end }} +{{- if $global.SSL.Engine }} + ssl-engine {{ $global.SSL.Engine }} +{{- if $global.SSL.ModeAsync }} + ssl-mode-async +{{- end }} +{{- end }} +{{- if $global.SSL.Ciphers }} + ssl-default-bind-ciphers {{ $global.SSL.Ciphers }} +{{- end }} +{{- if $global.SSL.Options }} + ssl-default-bind-options {{ $global.SSL.Options }} +{{- end }} +{{- range $snippet := $global.CustomConfig }} + {{ $snippet }} +{{- end }} + +defaults + log global +{{- if $global.LoadServerState }} + load-server-state-from-file global +{{- end }} + maxconn {{ $global.MaxConn }} +{{- if $global.DrainSupport }} + option persist +{{- else }} + option redispatch +{{- end }} + option dontlognull + option http-server-close + option http-keep-alive + timeout client {{ default "--" $global.Timeout.Client }} +{{- if $global.Timeout.ClientFin }} + timeout client-fin {{ $global.Timeout.ClientFin }} +{{- end }} + timeout connect {{ default "--" $global.Timeout.Connect }} +{{- if $global.Timeout.KeepAlive }} + timeout http-keep-alive {{ $global.Timeout.KeepAlive }} +{{- end }} +{{- if $global.Timeout.HTTPRequest }} + timeout http-request {{ $global.Timeout.HTTPRequest }} +{{- end }} +{{- if $global.Timeout.Queue }} + timeout queue {{ $global.Timeout.Queue }} +{{- end }} + timeout server {{ default "--" $global.Timeout.Server }} +{{- if $global.Timeout.ServerFin }} + timeout server-fin {{ $global.Timeout.ServerFin }} +{{- end }} +{{- if $global.Timeout.Tunnel }} + timeout tunnel {{ $global.Timeout.Tunnel }} +{{- end }} + + # # # # # # # # # # # # # # # # # # # +# # +# DNS RESOLVERS +# + +# TODO + +{{- $userlists := $cfg.Userlists }} +{{- if $userlists }} + + # # # # # # # # # # # # # # # # # # # +# # +# USER LISTS +# +{{- range $userlist := $userlists }} +userlist {{ $userlist.Name }} +{{- range $user := $userlist.Users }} + user {{ $user.Name }} {{ if not $user.Encrypted }}insecure-{{ end }}password {{ $user.Passwd }} +{{- end }} +{{- end }} +{{- end }} + + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# # TCP SERVICES +# # +# + +# TODO + + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# # BACKENDS +# # +# +{{- range $backend := $cfg.Backends }} +backend {{ $backend.ID }} + mode {{ if $backend.ModeTCP }}tcp{{ else }}http{{ end }} +{{- if $backend.BalanceAlgorithm }} + balance {{ $backend.BalanceAlgorithm }} +{{- end }} +{{- $timeout := $backend.Timeout }} +{{- if $timeout.Connect }} + timeout connect {{ $timeout.Connect }} +{{- end }} +{{- if $timeout.KeepAlive }} + timeout http-keep-alive {{ $timeout.KeepAlive }} +{{- end }} +{{- if $timeout.HTTPRequest }} + timeout http-request {{ $timeout.HTTPRequest }} +{{- end }} +{{- if $timeout.Queue }} + timeout queue {{ $timeout.Queue }} +{{- end }} +{{- if $timeout.Server }} + timeout server {{ $timeout.Server }} +{{- end }} +{{- if $timeout.ServerFin }} + timeout server-fin {{ $timeout.ServerFin }} +{{- end }} +{{- if $timeout.Tunnel }} + timeout tunnel {{ $timeout.Tunnel }} +{{- end }} + +{{- /*------------------------------------*/}} +{{- range $snippet := $backend.CustomConfig }} + {{ $snippet }} +{{- end }} + +{{- /*------------------------------------*/}} +{{- range $ep := $backend.Endpoints }} + server {{ $ep.Name }} {{ $ep.IP }}:{{ $ep.Port }} + {{- if $ep.Disabled }} disabled{{ end }} + {{- "" }} weight {{ $ep.Weight }} + {{- template "backend" map $backend }} +{{- end }} +{{- end }} + +{{- define "backend" }} + {{- $backend := .p1 }} + {{- if $backend.MaxConnServer }} maxconn {{ $backend.MaxConnServer }}{{ end }} + {{- if $backend.MaxQueueServer }} maxqueue {{ $backend.MaxQueueServer }}{{ end }} + {{- $ssl := $backend.SSL }} + {{- if $ssl.IsSecure }} ssl + {{- if $ssl.CertFilename }} crt {{ $ssl.CertFilename }}{{ end }} + {{- if $ssl.CAFilename }} verify required ca-file {{ $ssl.CAFilename }} + {{- else }} verify none + {{- end }} + {{- end }} + {{- if $backend.SendProxyProtocol }} {{ $backend.SendProxyProtocol }}{{ end }} + {{- $agent := $backend.AgentCheck }} + {{- $hc := $backend.HealthCheck }} + {{- if coalesce $hc.Port $hc.Addr $hc.Interval $hc.RiseCount $hc.FallCount }} check + {{- if $hc.Port }} port {{ $hc.Port }}{{ end }} + {{- if $hc.Addr }} addr {{ $hc.Addr }}{{ end }} + {{- if $hc.Interval }} inter {{ $hc.Interval }}{{ end }} + {{- if $hc.RiseCount }} rise {{ $hc.RiseCount }}{{ end }} + {{- if $hc.FallCount }} fall {{ $hc.FallCount }}{{ end }} + {{- end }} + {{- if $agent.Port }} agent-check agent-port {{ $agent.Port }} + {{- if $agent.Addr }} agent-addr {{ $agent.Addr }}{{ end }} + {{- if $agent.Interval }} agent-inter {{ $agent.Interval }}{{ end }} + {{- if $agent.Send }} agent-send {{ $agent.Send }}{{ end }} + {{- end }} +{{- end }} + + # # # # # # # # # # # # # # # # # # # +# # +# Error pages +# +{{- if and (not $cfg.DefaultHost) (not $cfg.DefaultBackend) }} +backend _error404 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/404.http + http-request deny deny_status 400 +{{- end }} +backend _error495 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/495.http + http-request deny deny_status 400 +backend _error496 + mode http + errorfile 400 /usr/local/etc/haproxy/errors/496.http + http-request deny deny_status 400 + + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # +# # FRONTENDS +# # +# +{{- $fgroup := $cfg.BuildFrontendGroup }} +{{- $frontends := $fgroup.Frontends }} +{{- if $fgroup.HasTCPProxy }} + + # # # # # # # # # # # # # # # # # # # +# # +# TCP/TLS frontend +# +listen _front__tls + mode tcp + bind :443 + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } +{{- if $fgroup.HasSSLPassthrough }} + ## ssl-passthrough + tcp-request content set-var(req.backend) req.ssl_sni,lower,map({{ $fgroup.SSLPassthroughMap }},_nomatch) + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } +{{- end }} +{{- range $frontend := $frontends }} + ## {{ $frontend.Name }} +{{- range $bind := $frontend.Binds }} + use-server _server_{{ $bind.Name }} if { req.ssl_sni -i + {{- range $host := $bind.Hosts }} {{ $host.Hostname }}{{ end }} } + server _server_{{ $bind.Name }} {{ $bind.Socket }} send-proxy-v2 +{{- end }} +{{- end }} + # TODO default backend +{{- end }} + + # # # # # # # # # # # # # # # # # # # +# # +# HTTP frontend +# +frontend _front__http + mode http + bind :80 + +{{- /*------------------------------------*/}} +{{- $hasredirect := $fgroup.HasRedirectHTTPS }} +{{- $hashttp := $fgroup.HasHTTPHost }} +{{- if $hasredirect }} + http-request set-var(req.base) base,regsub(:[0-9]+/,/) +{{- if $hashttp }} + http-request set-var(req.backend) var(req.base),map_beg({{ $fgroup.HTTPFrontsMap }},_nomatch) +{{- end }} +{{- else if $hashttp }} + http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg({{ $fgroup.HTTPFrontsMap }},_nomatch) +{{- end }} + +{{- /*------------------------------------*/}} +{{- if $hasredirect }} + redirect scheme https if { var(req.base) -i -m beg + {{- range $host := $cfg.Hosts }} + {{- range $path := $host.Paths }} + {{- if $path.Backend.SSLRedirect }} {{ $host.Hostname }}{{ $path.Path }}{{ end }} + {{- end }} + {{- end }} } +{{- end }} + +{{- /*------------------------------------*/}} +{{- if $hashttp }} + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } +{{- end }} + +{{- template "defaultbackend" map $cfg }} + + # # # # # # # # # # # # # # # # # # # +# # +# HTTPS frontends +# +{{- range $frontend := $frontends }} +frontend {{ $frontend.Name }} + mode http + +{{- /*------------------------------------*/}} +{{- range $bind := $frontend.Binds }} +{{- if $bind.Socket }} +{{- $tls := $bind.TLS }} + bind {{ $bind.Socket }} + {{- if $bind.AcceptProxy }} accept-proxy{{ end }} + {{- if or $tls.TLSCert $tls.TLSCertDir }} + {{- "" }} ssl alpn h2,http/1.1 + {{- if $tls.TLSCert }} crt {{ $tls.TLSCert }}{{ end }} + {{- if $tls.TLSCertDir }} crt {{ $tls.TLSCertDir }}{{ end }} + {{- end }} + {{- if $tls.CAFilename }} ca-file {{ $tls.CAFilename }} verify optional ca-ignore-err all crt-ignore-err all{{ end }} +{{- end }} +{{- end }} + +{{- /*------------------------------------*/}} +{{- if $frontend.Timeout.Client }} + timeout client {{ $frontend.Timeout.Client }} +{{- end }} +{{- if $frontend.Timeout.ClientFin }} + timeout client-fin {{ $frontend.Timeout.ClientFin }} +{{- end }} +{{- if $frontend.HasVarNamespace }} + http-request set-var(txn.namespace) base + {{- if $frontend.ConvertLowercase }},lower{{ end }} + {{- "" }},regsub(:[0-9]+/,/) + {{- "" }},map_beg({{ $frontend.VarNamespaceMap }},-) +{{- end }} + +{{- /*------------------------------------*/}} + http-request set-var(req.backend) base + {{- if $frontend.ConvertLowercase }},lower{{ end }} + {{- "" }},regsub(:[0-9]+/,/) + {{- "" }},map_beg({{ $frontend.BackendsMap }},_nomatch) + +{{- /*------------------------------------*/}} +{{- if $frontend.HasTLSAuth }} +{{- $optional := $frontend.HasTLSOptional }} + acl tls-invalid-crt ssl_c_ca_err gt 0 + acl tls-invalid-crt ssl_c_err gt 0 + acl tls-has-crt ssl_c_used + http-request set-var(req.tls_invalid_redir) ssl_fc_sni + {{- if $frontend.ConvertLowercase }},lower{{ end }} + {{- "" }},map({{ $frontend.TLSInvalidCrtErrorPagesMap }},_internal) + {{- "" }} if tls-invalid-crt +{{- if $optional }} + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni + {{- if $frontend.ConvertLowercase }},lower{{ end }} + {{- "" }},map({{ $frontend.TLSNoCrtErrorPagesMap }},_internal) + {{- "" }} if !tls-has-crt +{{- end }} +{{- if $frontend.HasInvalidErrorPage }} + redirect location %[var(req.tls_invalid_redir)] code 303 if + {{- "" }} { var(req.tls_invalid_redir) -m found } !{ var(req.tls_invalid_redir) _internal } +{{- end }} +{{- if and $optional $frontend.HasNoCrtErrorPage }} + redirect location %[var(req.tls_nocrt_redir)] code 303 if + {{- "" }} { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } +{{- end }} + use_backend _error495 if + {{- "" }} { var(req.tls_invalid_redir) _internal } + {{- "" }} { ssl_fc_sni -i + {{- range $host := $frontend.Hosts }} + {{- if $host.HasTLSAuth }} {{ $host.Hostname }}{{ end }} + {{- end }} } +{{- if $optional }} + use_backend _error496 if + {{- "" }} { var(req.tls_nocrt_redir) _internal } + {{- "" }} { ssl_fc_sni -i + {{- range $host := $frontend.Hosts }} + {{- if and $host.HasTLSAuth $host.TLS.CAVerifyOptional }} {{ $host.Hostname }}{{ end }} + {{- end }} } +{{- end }} +{{- end }} + +{{- /*------------------------------------*/}} + use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + +{{- template "defaultbackend" map $cfg }} +{{- end }} + +{{- /*------------------------------------*/}} +{{- /*------------------------------------*/}} +{{- define "defaultbackend" }} +{{- $cfg := .p1 }} +{{- if $cfg.DefaultHost }} +{{- range $path := $cfg.DefaultHost.Paths }} + use_backend {{ $path.BackendID }} + {{- if ne $path.Path "/" }} if { path_beg {{ $path.Path }} }{{ end }} +{{- end }} +{{- else if $cfg.DefaultBackend }} + default_backend {{ $cfg.DefaultBackend.ID }} +{{- else }} + default_backend _error404 +{{- end }} +{{- end }} diff --git a/rootfs/usr/local/etc/haproxy/errors/404.http b/rootfs/usr/local/etc/haproxy/errors/404.http new file mode 100644 index 000000000..b67e241e9 --- /dev/null +++ b/rootfs/usr/local/etc/haproxy/errors/404.http @@ -0,0 +1,8 @@ +HTTP/1.0 404 Not Found +Cache-Control: no-cache +Connection: close +Content-Type: text/html + +

404 Not Found

+The requested URL was not found. + From 411f604c4e457ce6500dad33f505b150c87a98b3 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Sun, 10 Mar 2019 08:10:44 -0300 Subject: [PATCH 7/8] add TLS authentication --- pkg/common/ingress/controller/controller.go | 15 +- pkg/controller/cache.go | 38 ++-- pkg/controller/controller.go | 39 +++- pkg/converters/ingress/annotations/global.go | 4 +- pkg/converters/ingress/annotations/host.go | 10 +- .../ingress/helper_test/cachemock.go | 30 +++- pkg/converters/ingress/ingress.go | 43 ++--- pkg/converters/ingress/ingress_test.go | 37 ++-- pkg/converters/ingress/types/annotations.go | 1 + pkg/converters/ingress/types/interfaces.go | 6 +- pkg/converters/ingress/types/options.go | 8 +- pkg/haproxy/config.go | 48 +++-- pkg/haproxy/helper_test/bindutilsmock.go | 4 +- pkg/haproxy/instance_test.go | 169 ++++++++++++------ pkg/haproxy/types/frontend.go | 6 +- pkg/haproxy/types/types.go | 3 +- rootfs/etc/haproxy/template/haproxy.tmpl | 58 +++--- 17 files changed, 318 insertions(+), 201 deletions(-) diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index c24a5a6e1..11ad53a7a 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -1532,15 +1532,17 @@ func (ic GenericController) Stop() error { return fmt.Errorf("shutdown already in progress") } +// StartControllers ... +func (ic *GenericController) StartControllers() { + ic.cacheController.Run(ic.stopCh) +} + // Start starts the Ingress controller. func (ic *GenericController) Start() { glog.Infof("starting Ingress controller") - ic.cacheController.Run(ic.stopCh) - - createDefaultSSLCertificate() - if ic.cfg.V07 { + ic.CreateDefaultSSLCertificate() time.Sleep(5 * time.Second) // initial sync of secrets to avoid unnecessary reloads glog.Info("running initial sync of secrets") @@ -1587,7 +1589,8 @@ func (ic *GenericController) SetForceReload(shouldReload bool) { } } -func createDefaultSSLCertificate() { +// CreateDefaultSSLCertificate ... +func (ic *GenericController) CreateDefaultSSLCertificate() (path, hash string) { defCert, defKey := ssl.GetFakeSSLCert() c, err := ssl.AddOrUpdateCertAndKey(fakeCertificate, defCert, defKey, []byte{}) if err != nil { @@ -1596,4 +1599,6 @@ func createDefaultSSLCertificate() { fakeCertificateSHA = c.PemSHA fakeCertificatePath = c.PemFileName + + return fakeCertificatePath, fakeCertificateSHA } diff --git a/pkg/controller/cache.go b/pkg/controller/cache.go index 81aae20f2..2c676cfdb 100644 --- a/pkg/controller/cache.go +++ b/pkg/controller/cache.go @@ -22,9 +22,11 @@ import ( api "k8s.io/api/core/v1" + "github.com/jcmoraisjr/haproxy-ingress/pkg/common/file" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/controller" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/net/ssl" + ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" ) type cache struct { @@ -56,44 +58,52 @@ func (c *cache) GetPod(podName string) (*api.Pod, error) { return c.listers.Pod.GetPod(sname[0], sname[1]) } -func (c *cache) GetTLSSecretPath(secretName string) (string, error) { +func (c *cache) GetTLSSecretPath(secretName string) (ingtypes.File, error) { sslCert, err := c.controller.GetCertificate(secretName) if err != nil { - return "", err + return ingtypes.File{}, err } if sslCert.PemFileName == "" { - return "", fmt.Errorf("secret '%s' does not have keys 'tls.crt' and 'tls.key'", secretName) + return ingtypes.File{}, fmt.Errorf("secret '%s' does not have keys 'tls.crt' and 'tls.key'", secretName) } - return sslCert.PemFileName, nil + return ingtypes.File{ + Filename: sslCert.PemFileName, + SHA1Hash: sslCert.PemSHA, + }, nil } -func (c *cache) GetCASecretPath(secretName string) (string, error) { +func (c *cache) GetCASecretPath(secretName string) (ingtypes.File, error) { sslCert, err := c.controller.GetCertificate(secretName) if err != nil { - return "", err + return ingtypes.File{}, err } if sslCert.CAFileName == "" { - return "", fmt.Errorf("secret '%s' does not have key 'ca.crt'", secretName) + return ingtypes.File{}, fmt.Errorf("secret '%s' does not have key 'ca.crt'", secretName) } - return sslCert.CAFileName, nil + return ingtypes.File{ + Filename: sslCert.CAFileName, + SHA1Hash: sslCert.PemSHA, + }, nil } -func (c *cache) GetDHSecretPath(secretName string) (string, error) { +func (c *cache) GetDHSecretPath(secretName string) (ingtypes.File, error) { secret, err := c.listers.Secret.GetByName(secretName) if err != nil { - return "", err + return ingtypes.File{}, err } dh, found := secret.Data[dhparamFilename] if !found { - return "", fmt.Errorf("secret '%s' does not have key '%s'", secretName, dhparamFilename) + return ingtypes.File{}, fmt.Errorf("secret '%s' does not have key '%s'", secretName, dhparamFilename) } pem := strings.Replace(secretName, "/", "_", -1) pemFileName, err := ssl.AddOrUpdateDHParam(pem, dh) if err != nil { - return "", fmt.Errorf("error creating dh-param file '%s': %v", pem, err) + return ingtypes.File{}, fmt.Errorf("error creating dh-param file '%s': %v", pem, err) } - // file.SHA1(pemFileName) - return pemFileName, nil + return ingtypes.File{ + Filename: pemFileName, + SHA1Hash: file.SHA1(pemFileName), + }, nil } func (c *cache) GetSecretContent(secretName, keyName string) ([]byte, error) { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c74dc1bc6..16bda7205 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -80,6 +80,7 @@ func (hc *HAProxyController) Info() *ingress.BackendInfo { // Start starts the controller func (hc *HAProxyController) Start() { hc.controller = controller.NewIngressController(hc) + hc.controller.StartControllers() hc.configController() hc.controller.Start() } @@ -96,13 +97,6 @@ func (hc *HAProxyController) configController() { // starting v0.8 only config logger := &logger{depth: 1} - hc.converterOptions = &ingtypes.ConverterOptions{ - Logger: logger, - Cache: newCache(hc.storeLister, hc.controller), - AnnotationPrefix: "ingress.kubernetes.io", - DefaultBackend: hc.cfg.DefaultService, - DefaultSSLSecret: hc.cfg.DefaultSSLCertificate, - } instanceOptions := haproxy.InstanceOptions{ HAProxyCmd: "haproxy", ReloadCmd: "/haproxy-reload.sh", @@ -114,6 +108,32 @@ func (hc *HAProxyController) configController() { if err := hc.instance.ParseTemplates(); err != nil { glog.Fatalf("error creating HAProxy instance: %v", err) } + cache := newCache(hc.storeLister, hc.controller) + hc.converterOptions = &ingtypes.ConverterOptions{ + Logger: logger, + Cache: cache, + AnnotationPrefix: "ingress.kubernetes.io", + DefaultBackend: hc.cfg.DefaultService, + DefaultSSLFile: hc.createDefaultSSLFile(cache), + } +} + +func (hc *HAProxyController) createDefaultSSLFile(cache *cache) (tlsFile ingtypes.File) { + if hc.cfg.DefaultSSLCertificate != "" { + tlsFile, err := cache.GetTLSSecretPath(hc.cfg.DefaultSSLCertificate) + if err == nil { + return tlsFile + } + glog.Warningf("using auto generated fake certificate due to an error reading default TLS certificate: %v", err) + } else { + glog.Info("using auto generated fake certificate") + } + path, hash := hc.controller.CreateDefaultSSLCertificate() + tlsFile = ingtypes.File{ + Filename: path, + SHA1Hash: hash, + } + return tlsFile } // CreateX509CertsDir hard link files from certs to a single directory. @@ -126,11 +146,12 @@ func (hc *HAProxyController) CreateX509CertsDir(bindName string, certs []string) return "", err } for _, cert := range certs { - file, err := os.Stat(cert) + srcFile, err := os.Stat(cert) if err != nil { return "", err } - if err := os.Link(cert, x509dir+"/"+file.Name()); err != os.ErrExist { + dstFile := x509dir + "/" + srcFile.Name() + if err := os.Link(cert, dstFile); err != nil { return "", err } } diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go index 1cde762ec..5b5d60828 100644 --- a/pkg/converters/ingress/annotations/global.go +++ b/pkg/converters/ingress/annotations/global.go @@ -90,8 +90,8 @@ func (c *updater) buildGlobalSSL(d *globalData) { d.global.SSL.Ciphers = d.config.SSLCiphers d.global.SSL.Options = d.config.SSLOptions if d.config.SSLDHParam != "" { - if dhFilename, err := c.cache.GetDHSecretPath(d.config.SSLDHParam); err == nil { - d.global.SSL.DHParam.Filename = dhFilename + if dhFile, err := c.cache.GetDHSecretPath(d.config.SSLDHParam); err == nil { + d.global.SSL.DHParam.Filename = dhFile.Filename } else { c.logger.Error("error reading DH params: %v", err) } diff --git a/pkg/converters/ingress/annotations/host.go b/pkg/converters/ingress/annotations/host.go index 6ed0b9084..bed6ace48 100644 --- a/pkg/converters/ingress/annotations/host.go +++ b/pkg/converters/ingress/annotations/host.go @@ -20,10 +20,18 @@ func (c *updater) buildHostAuthTLS(d *hostData) { if d.ann.AuthTLSSecret == "" { return } + verify := d.ann.AuthTLSVerifyClient + if verify == "off" { + return + } if cafile, err := c.cache.GetCASecretPath(d.ann.AuthTLSSecret); err == nil { - d.host.TLS.CAFilename = cafile + d.host.TLS.CAFilename = cafile.Filename + d.host.TLS.CAHash = cafile.SHA1Hash + d.host.TLS.CAVerifyOptional = verify == "optional" || verify == "optional_no_ca" d.host.TLS.CAErrorPage = d.ann.AuthTLSErrorPage d.host.TLS.AddCertHeader = d.ann.AuthTLSCertHeader + } else { + c.logger.Error("error building TLS auth config: %v", err) } } diff --git a/pkg/converters/ingress/helper_test/cachemock.go b/pkg/converters/ingress/helper_test/cachemock.go index 5399cdd10..b14139355 100644 --- a/pkg/converters/ingress/helper_test/cachemock.go +++ b/pkg/converters/ingress/helper_test/cachemock.go @@ -17,10 +17,13 @@ limitations under the License. package helper_test import ( + "crypto/sha1" "fmt" "strings" api "k8s.io/api/core/v1" + + ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" ) // SecretContent ... @@ -68,27 +71,36 @@ func (c *CacheMock) GetPod(podName string) (*api.Pod, error) { } // GetTLSSecretPath ... -func (c *CacheMock) GetTLSSecretPath(secretName string) (string, error) { +func (c *CacheMock) GetTLSSecretPath(secretName string) (ingtypes.File, error) { if path, found := c.SecretTLSPath[secretName]; found { - return path, nil + return ingtypes.File{ + Filename: path, + SHA1Hash: fmt.Sprintf("%x", sha1.Sum([]byte(path))), + }, nil } - return "", fmt.Errorf("secret not found: '%s'", secretName) + return ingtypes.File{}, fmt.Errorf("secret not found: '%s'", secretName) } // GetCASecretPath ... -func (c *CacheMock) GetCASecretPath(secretName string) (string, error) { +func (c *CacheMock) GetCASecretPath(secretName string) (ingtypes.File, error) { if path, found := c.SecretCAPath[secretName]; found { - return path, nil + return ingtypes.File{ + Filename: path, + SHA1Hash: fmt.Sprintf("%x", sha1.Sum([]byte(path))), + }, nil } - return "", fmt.Errorf("secret not found: '%s'", secretName) + return ingtypes.File{}, fmt.Errorf("secret not found: '%s'", secretName) } // GetDHSecretPath ... -func (c *CacheMock) GetDHSecretPath(secretName string) (string, error) { +func (c *CacheMock) GetDHSecretPath(secretName string) (ingtypes.File, error) { if path, found := c.SecretDHPath[secretName]; found { - return path, nil + return ingtypes.File{ + Filename: path, + SHA1Hash: fmt.Sprintf("%x", sha1.Sum([]byte(path))), + }, nil } - return "", fmt.Errorf("secret not found: '%s'", secretName) + return ingtypes.File{}, fmt.Errorf("secret not found: '%s'", secretName) } // GetSecretContent ... diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index b2f2fbd94..460d4de4b 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -48,6 +48,7 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con hostAnnotations: map[*hatypes.Host]*ingtypes.HostAnnotations{}, backendAnnotations: map[*hatypes.Backend]*ingtypes.BackendAnnotations{}, } + haproxy.ConfigDefaultX509Cert(options.DefaultSSLFile.Filename) if options.DefaultBackend != "" { if backend, err := c.addBackend(options.DefaultBackend, 0, &ingtypes.BackendAnnotations{}); err == nil { haproxy.ConfigDefaultBackend(backend) @@ -121,19 +122,16 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { for _, tls := range ing.Spec.TLS { for _, tlshost := range tls.Hosts { if tlshost == hostname { - tlsPath, err := c.addTLS(ing.Namespace, tls.SecretName) - if err == nil { - if host.TLS.TLSFilename == "" { - host.TLS.TLSFilename = tlsPath - } else if host.TLS.TLSFilename != tlsPath { - err = fmt.Errorf("TLS of host '%s' was already assigned", host.Hostname) - } - } - if err != nil { + tlsPath := c.addTLS(ing.Namespace, tls.SecretName) + if host.TLS.TLSHash == "" { + host.TLS.TLSFilename = tlsPath.Filename + host.TLS.TLSHash = tlsPath.SHA1Hash + } else if host.TLS.TLSHash != tlsPath.SHA1Hash { + msg := fmt.Sprintf("TLS of host '%s' was already assigned", host.Hostname) if tls.SecretName != "" { - c.logger.Warn("skipping TLS secret '%s' of ingress '%s': %v", tls.SecretName, fullIngName, err) + c.logger.Warn("skipping TLS secret '%s' of ingress '%s': %s", tls.SecretName, fullIngName, msg) } else { - c.logger.Warn("skipping default TLS secret of ingress '%s': %v", fullIngName, err) + c.logger.Warn("skipping default TLS secret of ingress '%s': %s", fullIngName, msg) } } } @@ -230,25 +228,16 @@ func (c *converter) addHTTPPassthrough(fullSvcName string, ingFrontAnn *ingtypes } } -func (c *converter) addTLS(namespace, secretName string) (string, error) { - defaultSecret := c.options.DefaultSSLSecret - tlsSecretName := defaultSecret +func (c *converter) addTLS(namespace, secretName string) ingtypes.File { if secretName != "" { - tlsSecretName = namespace + "/" + secretName - } - tlsPath, err := c.cache.GetTLSSecretPath(tlsSecretName) - if err != nil { - if tlsSecretName == defaultSecret { - return "", err - } - tlsSecretErr := err - tlsPath, err = c.cache.GetTLSSecretPath(defaultSecret) - if err != nil { - return "", fmt.Errorf("failed to use custom and default certificate. custom: %v; default: %v", tlsSecretErr, err) + tlsSecretName := namespace + "/" + secretName + tlsFile, err := c.cache.GetTLSSecretPath(tlsSecretName) + if err == nil { + return tlsFile } - c.logger.Warn("using default certificate due to an error reading secret '%s': %v", tlsSecretName, tlsSecretErr) + c.logger.Warn("using default certificate due to an error reading secret '%s': %v", tlsSecretName, err) } - return tlsPath, nil + return c.options.DefaultSSLFile } func (c *converter) addEndpoints(svc *api.Service, servicePort int, backend *hatypes.Backend) error { diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index f0dd5a514..d366b58ce 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -428,25 +428,7 @@ func TestSyncRedeclareTLSCustomFirst(t *testing.T) { WARN skipping default TLS secret of ingress 'default/echo2': TLS of host 'echo.example.com' was already assigned`) } -func TestSyncNoDefaultNoTLS(t *testing.T) { - c := setup(t) - defer c.teardown() - - c.cache.SecretTLSPath = map[string]string{} - c.createSvc1Auto() - c.Sync(c.createIngTLS1("default/echo", "echo.example.com", "/", "echo:8080", "")) - - c.compareConfigFront(` -- hostname: echo.example.com - paths: - - path: / - backend: default_echo_8080`) - - c.compareLogging(` -WARN skipping default TLS secret of ingress 'default/echo': secret not found: 'system/ingress-default'`) -} - -func TestSyncNoDefaultInvalidTLS(t *testing.T) { +func TestSyncInvalidTLS(t *testing.T) { c := setup(t) defer c.teardown() @@ -458,10 +440,12 @@ func TestSyncNoDefaultInvalidTLS(t *testing.T) { - hostname: echo.example.com paths: - path: / - backend: default_echo_8080`) + backend: default_echo_8080 + tls: + tlsfilename: /tls/tls-default.pem`) c.compareLogging(` -WARN skipping TLS secret 'tls-invalid' of ingress 'default/echo': failed to use custom and default certificate. custom: secret not found: 'default/tls-invalid'; default: secret not found: 'system/ingress-default'`) +WARN using default certificate due to an error reading secret 'default/tls-invalid': secret not found: 'default/tls-invalid'`) } func TestSyncRootPathDefault(t *testing.T) { @@ -977,10 +961,13 @@ var defaultBackendConfig = ` func (c *testConfig) SyncDef(config map[string]string, ing ...*extensions.Ingress) { conv := NewIngressConverter( &ingtypes.ConverterOptions{ - Cache: c.cache, - Logger: c.logger, - DefaultBackend: "system/default", - DefaultSSLSecret: "system/ingress-default", + Cache: c.cache, + Logger: c.logger, + DefaultBackend: "system/default", + DefaultSSLFile: ingtypes.File{ + Filename: "/tls/tls-default.pem", + SHA1Hash: "1", + }, AnnotationPrefix: "ingress.kubernetes.io", }, c.hconfig, diff --git a/pkg/converters/ingress/types/annotations.go b/pkg/converters/ingress/types/annotations.go index e57b6bbf3..9a4bf0925 100644 --- a/pkg/converters/ingress/types/annotations.go +++ b/pkg/converters/ingress/types/annotations.go @@ -22,6 +22,7 @@ type HostAnnotations struct { AppRoot string `json:"app-root"` AuthTLSCertHeader bool `json:"auth-tls-cert-header"` AuthTLSErrorPage string `json:"auth-tls-error-page"` + AuthTLSVerifyClient string `json:"auth-tls-verify-client"` AuthTLSSecret string `json:"auth-tls-secret"` ServerAlias string `json:"server-alias"` ServerAliasRegex string `json:"server-alias-regex"` diff --git a/pkg/converters/ingress/types/interfaces.go b/pkg/converters/ingress/types/interfaces.go index 08a875876..db60ab13f 100644 --- a/pkg/converters/ingress/types/interfaces.go +++ b/pkg/converters/ingress/types/interfaces.go @@ -25,8 +25,8 @@ type Cache interface { GetService(serviceName string) (*api.Service, error) GetEndpoints(service *api.Service) (*api.Endpoints, error) GetPod(podName string) (*api.Pod, error) - GetTLSSecretPath(secretName string) (string, error) - GetCASecretPath(secretName string) (string, error) - GetDHSecretPath(secretName string) (string, error) + GetTLSSecretPath(secretName string) (File, error) + GetCASecretPath(secretName string) (File, error) + GetDHSecretPath(secretName string) (File, error) GetSecretContent(secretName, keyName string) ([]byte, error) } diff --git a/pkg/converters/ingress/types/options.go b/pkg/converters/ingress/types/options.go index eac64d5cf..dcc49cbe7 100644 --- a/pkg/converters/ingress/types/options.go +++ b/pkg/converters/ingress/types/options.go @@ -20,11 +20,17 @@ import ( "github.com/jcmoraisjr/haproxy-ingress/pkg/types" ) +// File ... +type File struct { + Filename string + SHA1Hash string +} + // ConverterOptions ... type ConverterOptions struct { Logger types.Logger Cache Cache DefaultBackend string - DefaultSSLSecret string + DefaultSSLFile File AnnotationPrefix string } diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 97b418077..285fe5564 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -193,11 +193,12 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { } frontends, sslpassthrough := hatypes.BuildRawFrontends(c.hosts) for _, frontend := range frontends { - prefix := c.mapsDir + "/" + frontend.Name - frontend.BackendsMap = prefix + ".map" - frontend.TLSInvalidCrtErrorPagesMap = prefix + "_inv_crt.map" - frontend.TLSNoCrtErrorPagesMap = prefix + "_no_crt.map" - frontend.VarNamespaceMap = prefix + "_k8s_ns.map" + mapPrefix := c.mapsDir + "/" + frontend.Name + frontend.HostBackendsMap = mapPrefix + "_host.map" + frontend.SNIBackendsMap = mapPrefix + "_sni.map" + frontend.TLSInvalidCrtErrorPagesMap = mapPrefix + "_inv_crt.map" + frontend.TLSNoCrtErrorPagesMap = mapPrefix + "_no_crt.map" + frontend.VarNamespaceMap = mapPrefix + "_k8s_ns.map" } fgroup := &hatypes.FrontendGroup{ Frontends: frontends, @@ -214,7 +215,8 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { var bindName string if len(bind.Hosts) == 1 { bindName = bind.Hosts[0].Hostname - bind.TLS.TLSCert = bind.Hosts[0].TLS.TLSFilename + bind.TLS.TLSCert = c.defaultX509Cert + bind.TLS.TLSCertDir = bind.Hosts[0].TLS.TLSFilename } else { i++ bindName = fmt.Sprintf("_socket%03d", i) @@ -236,15 +238,15 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { bind.Name = "_public" bind.Socket = ":443" if len(bind.Hosts) == 1 { - bind.TLS.TLSCert = bind.Hosts[0].TLS.TLSFilename + bind.TLS.TLSCert = c.defaultX509Cert + bind.TLS.TLSCertDir = bind.Hosts[0].TLS.TLSFilename } else { x509dir, err := c.createCertsDir(bind.Name, bind.Hosts) if err != nil { return nil, err } - tls := &frontends[0].Binds[0].TLS - tls.TLSCert = c.defaultX509Cert - tls.TLSCertDir = x509dir + frontends[0].Binds[0].TLS.TLSCert = c.defaultX509Cert + frontends[0].Binds[0].TLS.TLSCertDir = x509dir } } type mapEntry struct { @@ -272,7 +274,8 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { } } for _, f := range frontends { - var backendsMap []mapEntry + var hostBackendsMap []mapEntry + var sniBackendsMap []mapEntry var invalidCrtMap []mapEntry var noCrtMap []mapEntry var varNamespaceMap []mapEntry @@ -282,7 +285,11 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { Key: host.Hostname + path.Path, Value: path.BackendID, } - backendsMap = append(backendsMap, entry) + if host.HasTLSAuth() { + sniBackendsMap = append(sniBackendsMap, entry) + } else { + hostBackendsMap = append(hostBackendsMap, entry) + } if path.Backend.SSLRedirect { fgroup.HasRedirectHTTPS = true } else { @@ -301,10 +308,15 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { Value: host.TLS.CAErrorPage, } invalidCrtMap = append(invalidCrtMap, entry) - noCrtMap = append(noCrtMap, entry) + if !host.TLS.CAVerifyOptional { + noCrtMap = append(noCrtMap, entry) + } } } - if err := c.mapsTemplate.WriteOutput(backendsMap, f.BackendsMap); err != nil { + if err := c.mapsTemplate.WriteOutput(hostBackendsMap, f.HostBackendsMap); err != nil { + return nil, err + } + if err := c.mapsTemplate.WriteOutput(sniBackendsMap, f.SNIBackendsMap); err != nil { return nil, err } if err := c.mapsTemplate.WriteOutput(invalidCrtMap, f.TLSInvalidCrtErrorPagesMap); err != nil { @@ -329,11 +341,17 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { func (c *config) createCertsDir(bindName string, hosts []*hatypes.Host) (string, error) { certs := make([]string, 0, len(hosts)) + added := map[string]bool{} for _, host := range hosts { - if host.TLS.TLSFilename != "" { + filename := host.TLS.TLSFilename + if filename != "" && !added[filename] && filename != c.defaultX509Cert { certs = append(certs, host.TLS.TLSFilename) + added[filename] = true } } + if len(certs) == 0 { + return "", nil + } return c.bindUtils.CreateX509CertsDir(bindName, certs) } diff --git a/pkg/haproxy/helper_test/bindutilsmock.go b/pkg/haproxy/helper_test/bindutilsmock.go index e2414a96a..9b17e4032 100644 --- a/pkg/haproxy/helper_test/bindutilsmock.go +++ b/pkg/haproxy/helper_test/bindutilsmock.go @@ -18,7 +18,7 @@ package helper_test // BindUtilsMock ... type BindUtilsMock struct { - CertDirs []*CertDir + CertDirs []CertDir } // CertDir ... @@ -30,7 +30,7 @@ type CertDir struct { // CreateX509CertsDir ... func (b *BindUtilsMock) CreateX509CertsDir(bindName string, certs []string) (string, error) { dir := "/var/haproxy/certs/" + bindName - b.CertDirs = append(b.CertDirs, &CertDir{ + b.CertDirs = append(b.CertDirs, CertDir{ Dir: dir, Certs: certs, }) diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index bb24be3eb..444e64c6b 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/kylelemons/godebug/diff" + yaml "gopkg.in/yaml.v2" ha_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/helper_test" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" @@ -82,9 +83,9 @@ frontend _front__http default_backend _error404 frontend https-front_empty mode http - bind :443 - http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_empty.map,_nomatch) - use_backend %%[var(req.backend)] unless { var(req.backend) _nomatch } + bind :443 ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem + http-request set-var(req.hostbackend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_empty_host.map,_nomatch) + use_backend %%[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } default_backend _error404 ` @@ -94,7 +95,7 @@ frontend https-front_empty c.checkMap("http-front.map", ` empty/ default_empty_8080`) - c.checkMap("https-front_empty.map", ` + c.checkMap("https-front_empty_host.map", ` empty/ default_empty_8080`) c.logger.CompareLogging(defaultLogging) @@ -117,7 +118,6 @@ func TestInstanceDefaultHost(t *testing.T) { h.AddPath(b, "/") b.SSLRedirect = true b.Endpoints = []*hatypes.Endpoint{endpointS1} - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" h.VarNamespace = true b = c.config.AcquireBackend("d2", "app", 8080) @@ -125,7 +125,6 @@ func TestInstanceDefaultHost(t *testing.T) { h.AddPath(b, "/app") b.SSLRedirect = true b.Endpoints = []*hatypes.Endpoint{endpointS1} - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" h.VarNamespace = true c.instance.Update() @@ -149,14 +148,14 @@ frontend https-front_d2.local mode http bind :443 ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem http-request set-var(txn.namespace) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d2.local_k8s_ns.map,-) - http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d2.local.map,_nomatch) - use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + http-request set-var(req.hostbackend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d2.local_host.map,_nomatch) + use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } use_backend d1_app_8080 `) c.checkMap("https-front_d2.local_k8s_ns.map", ` d2.local/app d2`) - c.checkMap("https-front_d2.local.map", ` + c.checkMap("https-front_d2.local_host.map", ` d2.local/app d2_app_8080`) c.logger.CompareLogging(defaultLogging) @@ -170,7 +169,6 @@ func TestInstanceSingleFrontendSingleBind(t *testing.T) { def := c.config.AcquireBackend("default", "default-backend", 8080) def.Endpoints = []*hatypes.Endpoint{endpointS0} c.config.ConfigDefaultBackend(def) - c.config.ConfigDefaultX509Cert("/var/haproxy/ssl/certs/default.pem") var h *hatypes.Host var b *hatypes.Backend @@ -213,18 +211,25 @@ frontend _front_001 mode http bind :443 ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem crt /var/haproxy/certs/_public http-request set-var(txn.namespace) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001_k8s_ns.map,-) - http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001.map,_nomatch) - use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + http-request set-var(req.hostbackend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001_host.map,_nomatch) + use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } default_backend _default_backend `) c.checkMap("_front_001_k8s_ns.map", ` d1.local/ d1 d2.local/app -`) - c.checkMap("_front_001.map", ` + c.checkMap("_front_001_host.map", ` d1.local/ d1_app_8080 d2.local/app d2_app_8080`) + c.checkCerts(` +certdirs: +- dir: /var/haproxy/certs/_public + certs: + - /var/haproxy/ssl/certs/d1.pem + - /var/haproxy/ssl/certs/d2.pem`) + c.logger.CompareLogging(defaultLogging) } @@ -245,14 +250,12 @@ func TestInstanceSingleFrontendTwoBindsCA(t *testing.T) { h.AddPath(b, "/") b.SSLRedirect = true b.Endpoints = []*hatypes.Endpoint{endpointS1} - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" h.TLS.CAFilename = "/var/haproxy/ssl/ca/d1.local.pem" h.TLS.CAHash = "1" h.TLS.CAErrorPage = "http://d1.local/error.html" h = c.config.AcquireHost("d2.local") h.AddPath(b, "/") - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" h.TLS.CAFilename = "/var/haproxy/ssl/ca/d2.local.pem" h.TLS.CAHash = "2" @@ -271,9 +274,9 @@ listen _front__tls tcp-request content accept if { req.ssl_hello_type 1 } ## _front_001 use-server _server_d1.local if { req.ssl_sni -i d1.local } - server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 + server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 weight 0 use-server _server_d2.local if { req.ssl_sni -i d2.local } - server _server_d2.local unix@/var/run/front_d2.local.sock send-proxy-v2 + server _server_d2.local unix@/var/run/front_d2.local.sock send-proxy-v2 weight 0 # TODO default backend frontend _front__http mode http @@ -285,14 +288,20 @@ frontend _front_001 mode http bind unix@/var/run/front_d1.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d1.local.pem verify optional ca-ignore-err all crt-ignore-err all bind unix@/var/run/front_d2.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d2.local.pem verify optional ca-ignore-err all crt-ignore-err all - http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001.map,_nomatch) + http-request set-var(req.hostbackend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001_host.map,_nomatch) + http-request set-header x-ha-base %[ssl_fc_sni]%[path] + http-request set-var(req.snibackend) hdr(x-ha-base),regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001_sni.map,_nomatch) acl tls-invalid-crt ssl_c_ca_err gt 0 acl tls-invalid-crt ssl_c_err gt 0 acl tls-has-crt ssl_c_used - http-request set-var(req.tls_invalid_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt - redirect location %[var(req.tls_invalid_redir)] code 303 if { var(req.tls_invalid_redir) -m found } !{ var(req.tls_invalid_redir) _internal } - use_backend _error495 if { var(req.tls_invalid_redir) _internal } { ssl_fc_sni -i d1.local d2.local } - use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_no_crt.map,_internal) if !tls-has-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt + http-request redirect location %[var(req.tls_nocrt_redir)] code 303 if { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } + http-request redirect location %[var(req.tls_invalidcrt_redir)] code 303 if { var(req.tls_invalidcrt_redir) -m found } !{ var(req.tls_invalidcrt_redir) _internal } + use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i d1.local d2.local } + use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i d1.local d2.local } + use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } + use_backend %[var(req.snibackend)] unless { var(req.snibackend) _nomatch } default_backend _default_backend `) @@ -300,7 +309,9 @@ frontend _front_001 d1.local http://d1.local/error.html`) c.checkMap("_front_001_no_crt.map", ` d1.local http://d1.local/error.html`) - c.checkMap("_front_001.map", ` + c.checkMap("_front_001_host.map", ` +`) + c.checkMap("_front_001_sni.map", ` d1.local/ d_app_8080 d2.local/ d_app_8080`) @@ -325,30 +336,39 @@ func TestInstanceTwoFrontendsThreeBindsCA(t *testing.T) { b.SSLRedirect = true b.Endpoints = []*hatypes.Endpoint{endpointS1} h.Timeout.Client = "1s" - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" h.TLS.CAFilename = "/var/haproxy/ssl/ca/d1.local.pem" h.TLS.CAHash = "1" h.TLS.CAVerifyOptional = true h.TLS.CAErrorPage = "http://d1.local/error.html" - h = c.config.AcquireHost("d2.local") + h = c.config.AcquireHost("d21.local") h.AddPath(b, "/") h.Timeout.Client = "2s" - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/d.pem" + h.TLS.TLSHash = "1" + h.TLS.CAFilename = "/var/haproxy/ssl/ca/d2.local.pem" + h.TLS.CAHash = "1" + h.TLS.CAVerifyOptional = true + h.TLS.CAErrorPage = "http://d21.local/error.html" + + h = c.config.AcquireHost("d22.local") + h.AddPath(b, "/") + h.Timeout.Client = "2s" + h.TLS.TLSFilename = "/var/haproxy/ssl/certs/d.pem" + h.TLS.TLSHash = "1" h.TLS.CAFilename = "/var/haproxy/ssl/ca/d2.local.pem" h.TLS.CAHash = "1" + h.TLS.CAErrorPage = "http://d22.local/error.html" b = c.config.AcquireBackend("d", "app", 8080) h = c.config.AcquireHost("d3.local") h.AddPath(b, "/") b.Endpoints = []*hatypes.Endpoint{endpointS21} h.Timeout.Client = "2s" - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" h = c.config.AcquireHost("d4.local") h.AddPath(b, "/") h.Timeout.Client = "2s" - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" c.instance.Update() c.checkConfig(` @@ -367,50 +387,56 @@ listen _front__tls tcp-request inspect-delay 5s tcp-request content accept if { req.ssl_hello_type 1 } ## _front_001 - use-server _server_d2.local if { req.ssl_sni -i d2.local } - server _server_d2.local unix@/var/run/front_d2.local.sock send-proxy-v2 - use-server _server__socket001 if { req.ssl_sni -i d3.local d4.local } - server _server__socket001 unix@/var/run/front__socket001.sock send-proxy-v2 + use-server _server__socket001 if { req.ssl_sni -i d21.local d22.local } + server _server__socket001 unix@/var/run/front__socket001.sock send-proxy-v2 weight 0 + use-server _server__socket002 if { req.ssl_sni -i d3.local d4.local } + server _server__socket002 unix@/var/run/front__socket002.sock send-proxy-v2 weight 0 ## https-front_d1.local use-server _server_d1.local if { req.ssl_sni -i d1.local } - server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 + server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 weight 0 # TODO default backend frontend _front__http mode http bind :80 http-request set-var(req.base) base,regsub(:[0-9]+/,/) http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) - redirect scheme https if { var(req.base) -i -m beg d1.local/ d2.local/ } + redirect scheme https if { var(req.base) -i -m beg d1.local/ d21.local/ d22.local/ } use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } default_backend _default_backend frontend _front_001 mode http - bind unix@/var/run/front_d2.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d2.local.pem verify optional ca-ignore-err all crt-ignore-err all - bind unix@/var/run/front__socket001.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/certs/_socket001 + bind unix@/var/run/front__socket001.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem crt /var/haproxy/certs/_socket001 ca-file /var/haproxy/ssl/ca/d2.local.pem verify optional ca-ignore-err all crt-ignore-err all + bind unix@/var/run/front__socket002.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem timeout client 2s - http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001.map,_nomatch) + http-request set-var(req.hostbackend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001_host.map,_nomatch) + http-request set-header x-ha-base %[ssl_fc_sni]%[path] + http-request set-var(req.snibackend) hdr(x-ha-base),regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/_front_001_sni.map,_nomatch) acl tls-invalid-crt ssl_c_ca_err gt 0 acl tls-invalid-crt ssl_c_err gt 0 acl tls-has-crt ssl_c_used - http-request set-var(req.tls_invalid_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt - use_backend _error495 if { var(req.tls_invalid_redir) _internal } { ssl_fc_sni -i d2.local } - use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_no_crt.map,_internal) if !tls-has-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt + http-request redirect location %[var(req.tls_nocrt_redir)] code 303 if { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } + http-request redirect location %[var(req.tls_invalidcrt_redir)] code 303 if { var(req.tls_invalidcrt_redir) -m found } !{ var(req.tls_invalidcrt_redir) _internal } + use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i d22.local } + use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i d21.local d22.local } + use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } + use_backend %[var(req.snibackend)] unless { var(req.snibackend) _nomatch } default_backend _default_backend frontend https-front_d1.local mode http bind unix@/var/run/front_d1.local.sock accept-proxy ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem ca-file /var/haproxy/ssl/ca/d1.local.pem verify optional ca-ignore-err all crt-ignore-err all timeout client 1s - http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d1.local.map,_nomatch) + http-request set-var(req.hostbackend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d1.local_host.map,_nomatch) + http-request set-header x-ha-base %[ssl_fc_sni]%[path] + http-request set-var(req.snibackend) hdr(x-ha-base),regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d1.local_sni.map,_nomatch) acl tls-invalid-crt ssl_c_ca_err gt 0 acl tls-invalid-crt ssl_c_err gt 0 - acl tls-has-crt ssl_c_used - http-request set-var(req.tls_invalid_redir) ssl_fc_sni,map(/etc/haproxy/maps/https-front_d1.local_inv_crt.map,_internal) if tls-invalid-crt - http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/https-front_d1.local_no_crt.map,_internal) if !tls-has-crt - redirect location %[var(req.tls_invalid_redir)] code 303 if { var(req.tls_invalid_redir) -m found } !{ var(req.tls_invalid_redir) _internal } - redirect location %[var(req.tls_nocrt_redir)] code 303 if { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } - use_backend _error495 if { var(req.tls_invalid_redir) _internal } { ssl_fc_sni -i d1.local } - use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i d1.local } - use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/https-front_d1.local_inv_crt.map,_internal) if tls-invalid-crt + http-request redirect location %[var(req.tls_invalidcrt_redir)] code 303 if { var(req.tls_invalidcrt_redir) -m found } !{ var(req.tls_invalidcrt_redir) _internal } + use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i d1.local } + use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } + use_backend %[var(req.snibackend)] unless { var(req.snibackend) _nomatch } default_backend _default_backend `) @@ -418,20 +444,33 @@ frontend https-front_d1.local d3.local/ d_app_8080 d4.local/ d_app_8080`) c.checkMap("_front_001_inv_crt.map", ` +d21.local http://d21.local/error.html +d22.local http://d22.local/error.html `) c.checkMap("_front_001_no_crt.map", ` +d22.local http://d22.local/error.html `) - c.checkMap("_front_001.map", ` -d2.local/ d_appca_8080 + c.checkMap("_front_001_host.map", ` d3.local/ d_app_8080 d4.local/ d_app_8080`) + c.checkMap("_front_001_sni.map", ` +d21.local/ d_appca_8080 +d22.local/ d_appca_8080`) c.checkMap("https-front_d1.local_inv_crt.map", ` d1.local http://d1.local/error.html`) c.checkMap("https-front_d1.local_no_crt.map", ` -d1.local http://d1.local/error.html`) - c.checkMap("https-front_d1.local.map", ` +`) + c.checkMap("https-front_d1.local_host.map", ` +`) + c.checkMap("https-front_d1.local_sni.map", ` d1.local/ d_appca_8080`) + c.checkCerts(` +certdirs: +- dir: /var/haproxy/certs/_socket001 + certs: + - /var/haproxy/ssl/certs/d.pem`) + c.logger.CompareLogging(defaultLogging) } @@ -452,7 +491,6 @@ func TestInstanceSomePaths(t *testing.T) { h.AddPath(b, "/") b.SSLRedirect = true b.Endpoints = []*hatypes.Endpoint{endpointS1} - h.TLS.TLSFilename = "/var/haproxy/ssl/certs/default.pem" b = c.config.AcquireBackend("d", "app1", 8080) h.AddPath(b, "/app") @@ -498,12 +536,12 @@ frontend _front__http frontend https-front_d.local mode http bind :443 ssl alpn h2,http/1.1 crt /var/haproxy/ssl/certs/default.pem - http-request set-var(req.backend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d.local.map,_nomatch) - use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } + http-request set-var(req.hostbackend) base,regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d.local_host.map,_nomatch) + use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } default_backend _default_backend `) - c.checkMap("https-front_d.local.map", ` + c.checkMap("https-front_d.local_host.map", ` d.local/sub d_app3_8080 d.local/app/sub d_app2_8080 d.local/app d_app1_8080 @@ -535,7 +573,6 @@ func TestSSLPassthrough(t *testing.T) { h.SSLPassthrough = true b = c.config.AcquireBackend("d3", "app-http", 8080) - b.SSLRedirect = true b.Endpoints = []*hatypes.Endpoint{endpointS41h} h.HTTPPassthroughBackend = b @@ -590,6 +627,7 @@ d3.local/ d3_app-http_8080`) type testConfig struct { t *testing.T logger *helper_test.LoggerMock + bindUtils *ha_helper.BindUtilsMock instance Instance config Config tempdir string @@ -624,14 +662,17 @@ func setup(t *testing.T) *testConfig { ); err != nil { t.Errorf("error parsing map.tmpl: %v", err) } - config := createConfig(&ha_helper.BindUtilsMock{}, options{ + bindUtils := &ha_helper.BindUtilsMock{} + config := createConfig(bindUtils, options{ mapsTemplate: instance.mapsTemplate, mapsDir: tempdir, }) instance.curConfig = config + config.ConfigDefaultX509Cert("/var/haproxy/ssl/certs/default.pem") return &testConfig{ t: t, logger: logger, + bindUtils: bindUtils, instance: instance, config: config, tempdir: tempdir, @@ -764,6 +805,11 @@ INFO (test) check was skipped INFO (test) reload was skipped INFO HAProxy successfully reloaded` +func _yamlMarshal(in interface{}) string { + out, _ := yaml.Marshal(in) + return string(out) +} + func (c *testConfig) checkConfig(backend, frontend string) { c.checkConfigFull(globalConfig + backend + errorPages + frontend) } @@ -778,6 +824,11 @@ func (c *testConfig) checkMap(mapName, expected string) { c.compareText(mapName, actual, expected) } +func (c *testConfig) checkCerts(expected string) { + actual := _yamlMarshal(c.bindUtils) + c.compareText("certs", actual, expected) +} + var replaceComments = regexp.MustCompile(`(?m)^[ \t]{0,2}(#.*)?[\r\n]+`) func (c *testConfig) readConfig(fileName string) string { diff --git a/pkg/haproxy/types/frontend.go b/pkg/haproxy/types/frontend.go index c4887a8b5..af6460988 100644 --- a/pkg/haproxy/types/frontend.go +++ b/pkg/haproxy/types/frontend.go @@ -60,10 +60,10 @@ func (f *Frontend) HasNoCrtErrorPage() bool { return f.HasInvalidErrorPage() } -// HasTLSOptional ... -func (f *Frontend) HasTLSOptional() bool { +// HasTLSMandatory ... +func (f *Frontend) HasTLSMandatory() bool { for _, host := range f.Hosts { - if host.TLS.CAVerifyOptional { + if !host.TLS.CAVerifyOptional { return true } } diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index b430bf45f..eea0dfcbb 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -99,8 +99,9 @@ type Frontend struct { Binds []*BindConfig Hosts []*Host // - BackendsMap string ConvertLowercase bool + HostBackendsMap string + SNIBackendsMap string Timeout HostTimeoutConfig TLSInvalidCrtErrorPagesMap string TLSNoCrtErrorPagesMap string diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index 530f4e5ea..994272be7 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -252,7 +252,7 @@ listen _front__tls {{- range $bind := $frontend.Binds }} use-server _server_{{ $bind.Name }} if { req.ssl_sni -i {{- range $host := $bind.Hosts }} {{ $host.Hostname }}{{ end }} } - server _server_{{ $bind.Name }} {{ $bind.Socket }} send-proxy-v2 + server _server_{{ $bind.Name }} {{ $bind.Socket }} send-proxy-v2 weight 0 {{- end }} {{- end }} # TODO default backend @@ -333,54 +333,62 @@ frontend {{ $frontend.Name }} {{- end }} {{- /*------------------------------------*/}} - http-request set-var(req.backend) base + http-request set-var(req.hostbackend) base {{- if $frontend.ConvertLowercase }},lower{{ end }} {{- "" }},regsub(:[0-9]+/,/) - {{- "" }},map_beg({{ $frontend.BackendsMap }},_nomatch) + {{- "" }},map_beg({{ $frontend.HostBackendsMap }},_nomatch) {{- /*------------------------------------*/}} {{- if $frontend.HasTLSAuth }} -{{- $optional := $frontend.HasTLSOptional }} +{{- /* missing concat converter, fix after 1.9 */}} + http-request set-header x-ha-base %[ssl_fc_sni]%[path] + http-request set-var(req.snibackend) hdr(x-ha-base) + {{- if $frontend.ConvertLowercase }},lower{{ end }} + {{- "" }},regsub(:[0-9]+/,/) + {{- "" }},map_beg({{ $frontend.SNIBackendsMap }},_nomatch) +{{- $mandatory := $frontend.HasTLSMandatory }} acl tls-invalid-crt ssl_c_ca_err gt 0 acl tls-invalid-crt ssl_c_err gt 0 +{{- if $mandatory }} acl tls-has-crt ssl_c_used - http-request set-var(req.tls_invalid_redir) ssl_fc_sni - {{- if $frontend.ConvertLowercase }},lower{{ end }} - {{- "" }},map({{ $frontend.TLSInvalidCrtErrorPagesMap }},_internal) - {{- "" }} if tls-invalid-crt -{{- if $optional }} http-request set-var(req.tls_nocrt_redir) ssl_fc_sni {{- if $frontend.ConvertLowercase }},lower{{ end }} {{- "" }},map({{ $frontend.TLSNoCrtErrorPagesMap }},_internal) {{- "" }} if !tls-has-crt {{- end }} -{{- if $frontend.HasInvalidErrorPage }} - redirect location %[var(req.tls_invalid_redir)] code 303 if - {{- "" }} { var(req.tls_invalid_redir) -m found } !{ var(req.tls_invalid_redir) _internal } -{{- end }} -{{- if and $optional $frontend.HasNoCrtErrorPage }} - redirect location %[var(req.tls_nocrt_redir)] code 303 if + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni + {{- if $frontend.ConvertLowercase }},lower{{ end }} + {{- "" }},map({{ $frontend.TLSInvalidCrtErrorPagesMap }},_internal) + {{- "" }} if tls-invalid-crt +{{- if and $mandatory $frontend.HasNoCrtErrorPage }} + http-request redirect location %[var(req.tls_nocrt_redir)] code 303 if {{- "" }} { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } {{- end }} - use_backend _error495 if - {{- "" }} { var(req.tls_invalid_redir) _internal } - {{- "" }} { ssl_fc_sni -i - {{- range $host := $frontend.Hosts }} - {{- if $host.HasTLSAuth }} {{ $host.Hostname }}{{ end }} - {{- end }} } -{{- if $optional }} +{{- if $frontend.HasInvalidErrorPage }} + http-request redirect location %[var(req.tls_invalidcrt_redir)] code 303 if + {{- "" }} { var(req.tls_invalidcrt_redir) -m found } !{ var(req.tls_invalidcrt_redir) _internal } +{{- end }} +{{- if $mandatory }} use_backend _error496 if {{- "" }} { var(req.tls_nocrt_redir) _internal } {{- "" }} { ssl_fc_sni -i {{- range $host := $frontend.Hosts }} - {{- if and $host.HasTLSAuth $host.TLS.CAVerifyOptional }} {{ $host.Hostname }}{{ end }} + {{- if and $host.HasTLSAuth (not $host.TLS.CAVerifyOptional) }} {{ $host.Hostname }}{{ end }} {{- end }} } {{- end }} + use_backend _error495 if + {{- "" }} { var(req.tls_invalidcrt_redir) _internal } + {{- "" }} { ssl_fc_sni -i + {{- range $host := $frontend.Hosts }} + {{- if $host.HasTLSAuth }} {{ $host.Hostname }}{{ end }} + {{- end }} } {{- end }} {{- /*------------------------------------*/}} - use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } - + use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } +{{- if $frontend.HasTLSAuth }} + use_backend %[var(req.snibackend)] unless { var(req.snibackend) _nomatch } +{{- end }} {{- template "defaultbackend" map $cfg }} {{- end }} From 8a6be4bd59052b684cbbd27afe99b8bd42f62581 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Tue, 12 Mar 2019 20:59:30 -0300 Subject: [PATCH 8/8] move all pattern lists from cfg to map or external lists --- pkg/haproxy/config.go | 72 ++++++++++++---- pkg/haproxy/instance_test.go | 102 +++++++++++++++++------ pkg/haproxy/types/types.go | 8 +- rootfs/etc/haproxy/maptemplate/map.tmpl | 2 +- rootfs/etc/haproxy/template/haproxy.tmpl | 22 ++--- 5 files changed, 145 insertions(+), 61 deletions(-) diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 285fe5564..2add7602d 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -192,18 +192,11 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { return nil, fmt.Errorf("cannot create frontends without hosts") } frontends, sslpassthrough := hatypes.BuildRawFrontends(c.hosts) - for _, frontend := range frontends { - mapPrefix := c.mapsDir + "/" + frontend.Name - frontend.HostBackendsMap = mapPrefix + "_host.map" - frontend.SNIBackendsMap = mapPrefix + "_sni.map" - frontend.TLSInvalidCrtErrorPagesMap = mapPrefix + "_inv_crt.map" - frontend.TLSNoCrtErrorPagesMap = mapPrefix + "_no_crt.map" - frontend.VarNamespaceMap = mapPrefix + "_k8s_ns.map" - } fgroup := &hatypes.FrontendGroup{ Frontends: frontends, HasSSLPassthrough: len(sslpassthrough) > 0, HTTPFrontsMap: c.mapsDir + "/http-front.map", + RedirectMap: c.mapsDir + "/redirect.map", SSLPassthroughMap: c.mapsDir + "/sslpassthrough.map", } if fgroup.HasTCPProxy() { @@ -249,12 +242,27 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { frontends[0].Binds[0].TLS.TLSCertDir = x509dir } } + for _, frontend := range frontends { + mapsPrefix := c.mapsDir + "/" + frontend.Name + frontend.HostBackendsMap = mapsPrefix + "_host.map" + frontend.SNIBackendsMap = mapsPrefix + "_sni.map" + frontend.TLSInvalidCrtErrorList = mapsPrefix + "_inv_crt.list" + frontend.TLSInvalidCrtErrorPagesMap = mapsPrefix + "_inv_crt_redir.map" + frontend.TLSNoCrtErrorList = mapsPrefix + "_no_crt.list" + frontend.TLSNoCrtErrorPagesMap = mapsPrefix + "_no_crt_redir.map" + frontend.VarNamespaceMap = mapsPrefix + "_k8s_ns.map" + for _, bind := range frontend.Binds { + bind.UseServerList = mapsPrefix + "_bind_" + bind.Name + ".list" + } + } type mapEntry struct { Key string Value string } var sslpassthroughMap []mapEntry + var redirectMap []mapEntry var httpFront []mapEntry + yesno := map[bool]string{true: "yes", false: "no"} for _, sslpassHost := range sslpassthrough { rootPath := sslpassHost.FindPath("/") if rootPath == nil { @@ -264,6 +272,10 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { Key: sslpassHost.Hostname, Value: rootPath.BackendID, }) + redirectMap = append(redirectMap, mapEntry{ + Key: sslpassHost.Hostname + "/", + Value: yesno[sslpassHost.HTTPPassthroughBackend == nil], + }) if sslpassHost.HTTPPassthroughBackend != nil { httpFront = append(httpFront, mapEntry{ Key: sslpassHost.Hostname + "/", @@ -276,11 +288,18 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { for _, f := range frontends { var hostBackendsMap []mapEntry var sniBackendsMap []mapEntry + var invalidCrtList []mapEntry var invalidCrtMap []mapEntry + var noCrtList []mapEntry var noCrtMap []mapEntry var varNamespaceMap []mapEntry for _, host := range f.Hosts { for _, path := range host.Paths { + // TODO use only root path if all uri has the same conf + redirectMap = append(redirectMap, mapEntry{ + Key: host.Hostname + path.Path, + Value: yesno[path.Backend.SSLRedirect], + }) entry := mapEntry{ Key: host.Hostname + path.Path, Value: path.BackendID, @@ -302,26 +321,46 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { } varNamespaceMap = append(varNamespaceMap, entry) } - if host.HasTLSAuth() && host.TLS.CAErrorPage != "" { - entry := mapEntry{ - Key: host.Hostname, - Value: host.TLS.CAErrorPage, - } - invalidCrtMap = append(invalidCrtMap, entry) + if host.HasTLSAuth() { + var entry mapEntry + entry.Key = host.Hostname + invalidCrtList = append(invalidCrtList, entry) if !host.TLS.CAVerifyOptional { - noCrtMap = append(noCrtMap, entry) + noCrtList = append(noCrtList, entry) + } + if host.TLS.CAErrorPage != "" { + entry.Value = host.TLS.CAErrorPage + invalidCrtMap = append(invalidCrtMap, entry) + if !host.TLS.CAVerifyOptional { + noCrtMap = append(noCrtMap, entry) + } } } } + for _, bind := range f.Binds { + var useServerList []mapEntry + for _, host := range bind.Hosts { + useServerList = append(useServerList, mapEntry{Key: host.Hostname}) + } + if err := c.mapsTemplate.WriteOutput(useServerList, bind.UseServerList); err != nil { + return nil, err + } + } if err := c.mapsTemplate.WriteOutput(hostBackendsMap, f.HostBackendsMap); err != nil { return nil, err } if err := c.mapsTemplate.WriteOutput(sniBackendsMap, f.SNIBackendsMap); err != nil { return nil, err } + if err := c.mapsTemplate.WriteOutput(invalidCrtList, f.TLSInvalidCrtErrorList); err != nil { + return nil, err + } if err := c.mapsTemplate.WriteOutput(invalidCrtMap, f.TLSInvalidCrtErrorPagesMap); err != nil { return nil, err } + if err := c.mapsTemplate.WriteOutput(noCrtList, f.TLSNoCrtErrorList); err != nil { + return nil, err + } if err := c.mapsTemplate.WriteOutput(noCrtMap, f.TLSNoCrtErrorPagesMap); err != nil { return nil, err } @@ -332,6 +371,9 @@ func (c *config) BuildFrontendGroup() (*hatypes.FrontendGroup, error) { if err := c.mapsTemplate.WriteOutput(sslpassthroughMap, fgroup.SSLPassthroughMap); err != nil { return nil, err } + if err := c.mapsTemplate.WriteOutput(redirectMap, fgroup.RedirectMap); err != nil { + return nil, err + } if err := c.mapsTemplate.WriteOutput(httpFront, fgroup.HTTPFrontsMap); err != nil { return nil, err } diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index 444e64c6b..a2d17c524 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -142,7 +142,7 @@ frontend _front__http mode http bind :80 http-request set-var(req.base) base,regsub(:[0-9]+/,/) - redirect scheme https if { var(req.base) -i -m beg d2.local/app } + redirect scheme https if { var(req.base),map_beg(/etc/haproxy/maps/redirect.map,_nomatch) yes } use_backend d1_app_8080 frontend https-front_d2.local mode http @@ -157,6 +157,8 @@ frontend https-front_d2.local d2.local/app d2`) c.checkMap("https-front_d2.local_host.map", ` d2.local/app d2_app_8080`) + c.checkMap("redirect.map", ` +d2.local/app yes`) c.logger.CompareLogging(defaultLogging) } @@ -205,7 +207,7 @@ frontend _front__http mode http bind :80 http-request set-var(req.base) base,regsub(:[0-9]+/,/) - redirect scheme https if { var(req.base) -i -m beg d1.local/ d2.local/app } + redirect scheme https if { var(req.base),map_beg(/etc/haproxy/maps/redirect.map,_nomatch) yes } default_backend _default_backend frontend _front_001 mode http @@ -222,6 +224,9 @@ d2.local/app -`) c.checkMap("_front_001_host.map", ` d1.local/ d1_app_8080 d2.local/app d2_app_8080`) + c.checkMap("redirect.map", ` +d1.local/ yes +d2.local/app yes`) c.checkCerts(` certdirs: @@ -273,16 +278,16 @@ listen _front__tls tcp-request inspect-delay 5s tcp-request content accept if { req.ssl_hello_type 1 } ## _front_001 - use-server _server_d1.local if { req.ssl_sni -i d1.local } + use-server _server_d1.local if { req.ssl_sni -i -f /etc/haproxy/maps/_front_001_bind_d1.local.list } server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 weight 0 - use-server _server_d2.local if { req.ssl_sni -i d2.local } + use-server _server_d2.local if { req.ssl_sni -i -f /etc/haproxy/maps/_front_001_bind_d2.local.list } server _server_d2.local unix@/var/run/front_d2.local.sock send-proxy-v2 weight 0 # TODO default backend frontend _front__http mode http bind :80 http-request set-var(req.base) base,regsub(:[0-9]+/,/) - redirect scheme https if { var(req.base) -i -m beg d1.local/ d2.local/ } + redirect scheme https if { var(req.base),map_beg(/etc/haproxy/maps/redirect.map,_nomatch) yes } default_backend _default_backend frontend _front_001 mode http @@ -294,26 +299,39 @@ frontend _front_001 acl tls-invalid-crt ssl_c_ca_err gt 0 acl tls-invalid-crt ssl_c_err gt 0 acl tls-has-crt ssl_c_used - http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_no_crt.map,_internal) if !tls-has-crt - http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_no_crt_redir.map,_internal) if !tls-has-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt_redir.map,_internal) if tls-invalid-crt http-request redirect location %[var(req.tls_nocrt_redir)] code 303 if { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } http-request redirect location %[var(req.tls_invalidcrt_redir)] code 303 if { var(req.tls_invalidcrt_redir) -m found } !{ var(req.tls_invalidcrt_redir) _internal } - use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i d1.local d2.local } - use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i d1.local d2.local } + use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i -f /etc/haproxy/maps/_front_001_no_crt.list } + use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i -f /etc/haproxy/maps/_front_001_inv_crt.list } use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } use_backend %[var(req.snibackend)] unless { var(req.snibackend) _nomatch } default_backend _default_backend `) - c.checkMap("_front_001_inv_crt.map", ` + c.checkMap("_front_001_inv_crt_redir.map", ` d1.local http://d1.local/error.html`) - c.checkMap("_front_001_no_crt.map", ` + c.checkMap("_front_001_inv_crt.list", ` +d1.local +d2.local`) + c.checkMap("_front_001_no_crt_redir.map", ` d1.local http://d1.local/error.html`) + c.checkMap("_front_001_no_crt.list", ` +d1.local +d2.local`) c.checkMap("_front_001_host.map", ` `) c.checkMap("_front_001_sni.map", ` d1.local/ d_app_8080 d2.local/ d_app_8080`) + c.checkMap("_front_001_bind_d1.local.list", ` +d1.local`) + c.checkMap("_front_001_bind_d2.local.list", ` +d2.local`) + c.checkMap("redirect.map", ` +d1.local/ yes +d2.local/ yes`) c.logger.CompareLogging(defaultLogging) } @@ -387,12 +405,12 @@ listen _front__tls tcp-request inspect-delay 5s tcp-request content accept if { req.ssl_hello_type 1 } ## _front_001 - use-server _server__socket001 if { req.ssl_sni -i d21.local d22.local } + use-server _server__socket001 if { req.ssl_sni -i -f /etc/haproxy/maps/_front_001_bind__socket001.list } server _server__socket001 unix@/var/run/front__socket001.sock send-proxy-v2 weight 0 - use-server _server__socket002 if { req.ssl_sni -i d3.local d4.local } + use-server _server__socket002 if { req.ssl_sni -i -f /etc/haproxy/maps/_front_001_bind__socket002.list } server _server__socket002 unix@/var/run/front__socket002.sock send-proxy-v2 weight 0 ## https-front_d1.local - use-server _server_d1.local if { req.ssl_sni -i d1.local } + use-server _server_d1.local if { req.ssl_sni -i -f /etc/haproxy/maps/https-front_d1.local_bind_d1.local.list } server _server_d1.local unix@/var/run/front_d1.local.sock send-proxy-v2 weight 0 # TODO default backend frontend _front__http @@ -400,7 +418,7 @@ frontend _front__http bind :80 http-request set-var(req.base) base,regsub(:[0-9]+/,/) http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) - redirect scheme https if { var(req.base) -i -m beg d1.local/ d21.local/ d22.local/ } + redirect scheme https if { var(req.base),map_beg(/etc/haproxy/maps/redirect.map,_nomatch) yes } use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } default_backend _default_backend frontend _front_001 @@ -414,12 +432,12 @@ frontend _front_001 acl tls-invalid-crt ssl_c_ca_err gt 0 acl tls-invalid-crt ssl_c_err gt 0 acl tls-has-crt ssl_c_used - http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_no_crt.map,_internal) if !tls-has-crt - http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt.map,_internal) if tls-invalid-crt + http-request set-var(req.tls_nocrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_no_crt_redir.map,_internal) if !tls-has-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/_front_001_inv_crt_redir.map,_internal) if tls-invalid-crt http-request redirect location %[var(req.tls_nocrt_redir)] code 303 if { var(req.tls_nocrt_redir) -m found } !{ var(req.tls_nocrt_redir) _internal } http-request redirect location %[var(req.tls_invalidcrt_redir)] code 303 if { var(req.tls_invalidcrt_redir) -m found } !{ var(req.tls_invalidcrt_redir) _internal } - use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i d22.local } - use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i d21.local d22.local } + use_backend _error496 if { var(req.tls_nocrt_redir) _internal } { ssl_fc_sni -i -f /etc/haproxy/maps/_front_001_no_crt.list } + use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i -f /etc/haproxy/maps/_front_001_inv_crt.list } use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } use_backend %[var(req.snibackend)] unless { var(req.snibackend) _nomatch } default_backend _default_backend @@ -432,9 +450,9 @@ frontend https-front_d1.local http-request set-var(req.snibackend) hdr(x-ha-base),regsub(:[0-9]+/,/),map_beg(/etc/haproxy/maps/https-front_d1.local_sni.map,_nomatch) acl tls-invalid-crt ssl_c_ca_err gt 0 acl tls-invalid-crt ssl_c_err gt 0 - http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/https-front_d1.local_inv_crt.map,_internal) if tls-invalid-crt + http-request set-var(req.tls_invalidcrt_redir) ssl_fc_sni,map(/etc/haproxy/maps/https-front_d1.local_inv_crt_redir.map,_internal) if tls-invalid-crt http-request redirect location %[var(req.tls_invalidcrt_redir)] code 303 if { var(req.tls_invalidcrt_redir) -m found } !{ var(req.tls_invalidcrt_redir) _internal } - use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i d1.local } + use_backend _error495 if { var(req.tls_invalidcrt_redir) _internal } { ssl_fc_sni -i -f /etc/haproxy/maps/https-front_d1.local_inv_crt.list } use_backend %[var(req.hostbackend)] unless { var(req.hostbackend) _nomatch } use_backend %[var(req.snibackend)] unless { var(req.snibackend) _nomatch } default_backend _default_backend @@ -443,27 +461,49 @@ frontend https-front_d1.local c.checkMap("http-front.map", ` d3.local/ d_app_8080 d4.local/ d_app_8080`) - c.checkMap("_front_001_inv_crt.map", ` + c.checkMap("_front_001_inv_crt_redir.map", ` d21.local http://d21.local/error.html d22.local http://d22.local/error.html `) - c.checkMap("_front_001_no_crt.map", ` + c.checkMap("_front_001_inv_crt.list", ` +d21.local +d22.local`) + c.checkMap("_front_001_no_crt_redir.map", ` d22.local http://d22.local/error.html `) + c.checkMap("_front_001_no_crt.list", ` +d22.local`) c.checkMap("_front_001_host.map", ` d3.local/ d_app_8080 d4.local/ d_app_8080`) c.checkMap("_front_001_sni.map", ` d21.local/ d_appca_8080 d22.local/ d_appca_8080`) - c.checkMap("https-front_d1.local_inv_crt.map", ` + c.checkMap("_front_001_bind__socket001.list", ` +d21.local +d22.local`) + c.checkMap("_front_001_bind__socket002.list", ` +d3.local +d4.local`) + c.checkMap("https-front_d1.local_inv_crt_redir.map", ` d1.local http://d1.local/error.html`) - c.checkMap("https-front_d1.local_no_crt.map", ` + c.checkMap("https-front_d1.local_inv_crt.list", ` +d1.local`) + c.checkMap("https-front_d1.local_no_crt_redir.map", ` `) c.checkMap("https-front_d1.local_host.map", ` `) c.checkMap("https-front_d1.local_sni.map", ` d1.local/ d_appca_8080`) + c.checkMap("https-front_d1.local_bind_d1.local.list", ` +d1.local`) + c.checkMap("redirect.map", ` +d21.local/ yes +d22.local/ yes +d3.local/ no +d4.local/ no +d1.local/ yes +`) c.checkCerts(` certdirs: @@ -530,7 +570,7 @@ frontend _front__http bind :80 http-request set-var(req.base) base,regsub(:[0-9]+/,/) http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) - redirect scheme https if { var(req.base) -i -m beg d.local/app d.local/ } + redirect scheme https if { var(req.base),map_beg(/etc/haproxy/maps/redirect.map,_nomatch) yes } use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } default_backend _default_backend frontend https-front_d.local @@ -546,6 +586,11 @@ d.local/sub d_app3_8080 d.local/app/sub d_app2_8080 d.local/app d_app1_8080 d.local/ d_app0_8080`) + c.checkMap("redirect.map", ` +d.local/sub no +d.local/app/sub no +d.local/app yes +d.local/ yes`) c.logger.CompareLogging(defaultLogging) } @@ -605,7 +650,7 @@ frontend _front__http bind :80 http-request set-var(req.base) base,regsub(:[0-9]+/,/) http-request set-var(req.backend) var(req.base),map_beg(/etc/haproxy/maps/http-front.map,_nomatch) - redirect scheme https if { var(req.base) -i -m beg d2.local/ } + redirect scheme https if { var(req.base),map_beg(/etc/haproxy/maps/redirect.map,_nomatch) yes } use_backend %[var(req.backend)] unless { var(req.backend) _nomatch } default_backend _error404`) @@ -614,6 +659,9 @@ d2.local d2_app_8080 d3.local d3_app-ssl_8443`) c.checkMap("http-front.map", ` d3.local/ d3_app-http_8080`) + c.checkMap("redirect.map", ` +d2.local/ yes +d3.local/ no`) c.logger.CompareLogging(defaultLogging) } diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index eea0dfcbb..4457799bf 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -90,6 +90,7 @@ type FrontendGroup struct { HasRedirectHTTPS bool HasSSLPassthrough bool HTTPFrontsMap string + RedirectMap string SSLPassthroughMap string } @@ -103,6 +104,8 @@ type Frontend struct { HostBackendsMap string SNIBackendsMap string Timeout HostTimeoutConfig + TLSInvalidCrtErrorList string + TLSNoCrtErrorList string TLSInvalidCrtErrorPagesMap string TLSNoCrtErrorPagesMap string VarNamespaceMap string @@ -114,8 +117,9 @@ type BindConfig struct { Socket string Hosts []*Host // - AcceptProxy bool - TLS BindTLSConfig + AcceptProxy bool + TLS BindTLSConfig + UseServerList string } // BindTLSConfig ... diff --git a/rootfs/etc/haproxy/maptemplate/map.tmpl b/rootfs/etc/haproxy/maptemplate/map.tmpl index f2ad043ce..2038b7213 100644 --- a/rootfs/etc/haproxy/maptemplate/map.tmpl +++ b/rootfs/etc/haproxy/maptemplate/map.tmpl @@ -7,5 +7,5 @@ # # # {{- range . }} -{{ .Key }} {{ .Value }} +{{ .Key }}{{ if .Value }} {{ .Value }}{{ end }} {{- end }} diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index 994272be7..8d0a1f01c 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -250,8 +250,8 @@ listen _front__tls {{- range $frontend := $frontends }} ## {{ $frontend.Name }} {{- range $bind := $frontend.Binds }} - use-server _server_{{ $bind.Name }} if { req.ssl_sni -i - {{- range $host := $bind.Hosts }} {{ $host.Hostname }}{{ end }} } + use-server _server_{{ $bind.Name }} if + {{- "" }} { req.ssl_sni -i -f {{ $bind.UseServerList }} } server _server_{{ $bind.Name }} {{ $bind.Socket }} send-proxy-v2 weight 0 {{- end }} {{- end }} @@ -280,12 +280,8 @@ frontend _front__http {{- /*------------------------------------*/}} {{- if $hasredirect }} - redirect scheme https if { var(req.base) -i -m beg - {{- range $host := $cfg.Hosts }} - {{- range $path := $host.Paths }} - {{- if $path.Backend.SSLRedirect }} {{ $host.Hostname }}{{ $path.Path }}{{ end }} - {{- end }} - {{- end }} } + redirect scheme https if + {{- "" }} { var(req.base),map_beg({{ $fgroup.RedirectMap }},_nomatch) yes } {{- end }} {{- /*------------------------------------*/}} @@ -371,17 +367,11 @@ frontend {{ $frontend.Name }} {{- if $mandatory }} use_backend _error496 if {{- "" }} { var(req.tls_nocrt_redir) _internal } - {{- "" }} { ssl_fc_sni -i - {{- range $host := $frontend.Hosts }} - {{- if and $host.HasTLSAuth (not $host.TLS.CAVerifyOptional) }} {{ $host.Hostname }}{{ end }} - {{- end }} } + {{- "" }} { ssl_fc_sni -i -f {{ $frontend.TLSNoCrtErrorList }} } {{- end }} use_backend _error495 if {{- "" }} { var(req.tls_invalidcrt_redir) _internal } - {{- "" }} { ssl_fc_sni -i - {{- range $host := $frontend.Hosts }} - {{- if $host.HasTLSAuth }} {{ $host.Hostname }}{{ end }} - {{- end }} } + {{- "" }} { ssl_fc_sni -i -f {{ $frontend.TLSInvalidCrtErrorList }} } {{- end }} {{- /*------------------------------------*/}}