diff --git a/go.mod b/go.mod index ed631da56..18cf5ea50 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/huandu/xstrings v1.2.1 // indirect github.com/imdario/mergo v0.3.5 + github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/json-iterator/go v1.1.8 // indirect github.com/kylelemons/godebug v1.1.0 github.com/mitchellh/copystructure v1.0.0 // indirect diff --git a/go.sum b/go.sum index b9249628a..6a6814fb6 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/huandu/xstrings v1.2.1 h1:v6IdmkCnDhJG/S0ivr58PeIfg+tyhqQYy4YsCsQ0Pdc github.com/huandu/xstrings v1.2.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -198,7 +200,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/pkg/common/ingress/controller/backend_ssl.go b/pkg/common/ingress/controller/backend_ssl.go index 2af452b7e..4de3979aa 100644 --- a/pkg/common/ingress/controller/backend_ssl.go +++ b/pkg/common/ingress/controller/backend_ssl.go @@ -43,7 +43,6 @@ func (ic *GenericController) SyncSecret(key string) { cert, err := ic.getPemCertificate(secret) if err != nil { glog.V(3).Infof("syncing a non ca/crt secret %v", key) - ic.newctrl.Notify() return } @@ -59,15 +58,13 @@ func (ic *GenericController) SyncSecret(key string) { ic.sslCertTracker.Update(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) - ic.newctrl.Notify() return } - glog.Infof("adding secret %v to the local store", key) + glog.V(3).Infof("adding secret %v to the local store", key) ic.sslCertTracker.Add(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) - ic.newctrl.Notify() } // getPemCertificate receives a secret, and creates a ingress.SSLCert as return. diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index d67d7c0e0..345a7fdc8 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -40,7 +40,6 @@ import ( type NewCtrlIntf interface { GetIngressList() ([]*networking.Ingress, error) GetSecret(name string) (*apiv1.Secret, error) - Notify() } // GenericController holds the boilerplate code required to build an Ingress controlller. @@ -196,6 +195,13 @@ func (ic GenericController) GetFullResourceName(name, currentNamespace string) s return name } +// UpdateSecret ... +func (ic GenericController) UpdateSecret(key string) { + if _, found := ic.sslCertTracker.Get(key); found { + ic.SyncSecret(key) + } +} + // DeleteSecret ... func (ic GenericController) DeleteSecret(key string) { ic.sslCertTracker.DeleteAll(key) diff --git a/pkg/controller/cache.go b/pkg/controller/cache.go index 6bb1b7d5d..f1228a95f 100644 --- a/pkg/controller/cache.go +++ b/pkg/controller/cache.go @@ -25,18 +25,26 @@ import ( "fmt" "os" "strings" + "sync" + "time" api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" "github.com/jcmoraisjr/haproxy-ingress/pkg/acme" cfile "github.com/jcmoraisjr/haproxy-ingress/pkg/common/file" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/controller" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/net/ssl" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + "github.com/jcmoraisjr/haproxy-ingress/pkg/types" + "github.com/jcmoraisjr/haproxy-ingress/pkg/utils" ) const dhparamFilename = "dhparam.pem" @@ -45,12 +53,47 @@ type k8scache struct { client k8s.Interface listers *listers controller *controller.GenericController + tracker convtypes.Tracker crossNS bool + globalConfigMapKey string + tcpConfigMapKey string acmeSecretKeyName string acmeTokenConfigmapName string + // + updateQueue utils.Queue + stateMutex sync.RWMutex + clear bool + needFullSync bool + // + globalConfigMapData map[string]string + tcpConfigMapData map[string]string + globalConfigMapDataNew map[string]string + tcpConfigMapDataNew map[string]string + // + ingressesDel []*networking.Ingress + ingressesUpd []*networking.Ingress + ingressesAdd []*networking.Ingress + endpointsNew []*api.Endpoints + servicesDel []*api.Service + servicesUpd []*api.Service + servicesAdd []*api.Service + secretsDel []*api.Secret + secretsUpd []*api.Secret + secretsAdd []*api.Secret + podsNew []*api.Pod + // } -func newCache(client k8s.Interface, listers *listers, controller *controller.GenericController) *k8scache { +func createCache( + logger types.Logger, + client k8s.Interface, + controller *controller.GenericController, + tracker convtypes.Tracker, + updateQueue utils.Queue, + watchNamespace string, + isolateNamespace bool, + resync time.Duration, +) *k8scache { namespace := os.Getenv("POD_NAMESPACE") if namespace == "" { // TODO implement a smart fallback or error checking @@ -70,14 +113,37 @@ func newCache(client k8s.Interface, listers *listers, controller *controller.Gen if !strings.Contains(acmeTokenConfigmapName, "/") { acmeTokenConfigmapName = namespace + "/" + acmeTokenConfigmapName } - return &k8scache{ + globalConfigMapName := cfg.ConfigMapName + tcpConfigMapName := cfg.TCPConfigMapName + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(logger.Info) + eventBroadcaster.StartRecordingToSink(&typedv1.EventSinkImpl{ + Interface: client.CoreV1().Events(watchNamespace), + }) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, api.EventSource{ + Component: "ingress-controller", + }) + cache := &k8scache{ client: client, - listers: listers, controller: controller, + tracker: tracker, crossNS: cfg.AllowCrossNamespace, + globalConfigMapKey: globalConfigMapName, + tcpConfigMapKey: tcpConfigMapName, acmeSecretKeyName: acmeSecretKeyName, acmeTokenConfigmapName: acmeTokenConfigmapName, - } + stateMutex: sync.RWMutex{}, + updateQueue: updateQueue, + clear: true, + needFullSync: false, + } + // TODO I'm a circular reference, can you fix me? + cache.listers = createListers(cache, logger, recorder, client, watchNamespace, isolateNamespace, resync) + return cache +} + +func (c *k8scache) RunAsync(stopCh <-chan struct{}) { + c.listers.RunAsync(stopCh) } func (c *k8scache) GetIngressPodName() (namespace, podname string, err error) { @@ -92,6 +158,34 @@ func (c *k8scache) GetIngressPodName() (namespace, podname string, err error) { return namespace, podname, nil } +func (c *k8scache) GetIngress(ingressName string) (*networking.Ingress, error) { + namespace, name, err := cache.SplitMetaNamespaceKey(ingressName) + if err != nil { + return nil, err + } + ing, err := c.listers.ingressLister.Ingresses(namespace).Get(name) + if ing != nil && !c.IsValidIngress(ing) { + return nil, fmt.Errorf("ingress class does not match") + } + return ing, err +} + +func (c *k8scache) GetIngressList() ([]*networking.Ingress, error) { + ingList, err := c.listers.ingressLister.List(labels.Everything()) + if err != nil { + return nil, err + } + validIngList := make([]*networking.Ingress, len(ingList)) + var i int + for _, ing := range ingList { + if c.IsValidIngress(ing) { + validIngList[i] = ing + i++ + } + } + return validIngList[:i], nil +} + func (c *k8scache) GetService(serviceName string) (*api.Service, error) { namespace, name, err := cache.SplitMetaNamespaceKey(serviceName) if err != nil { @@ -105,7 +199,10 @@ func (c *k8scache) GetSecret(secretName string) (*api.Secret, error) { if err != nil { return nil, err } - return c.listers.secretLister.Secrets(namespace).Get(name) + if c.listers.running { + return c.listers.secretLister.Secrets(namespace).Get(name) + } + return c.client.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{}) } func (c *k8scache) GetConfigMap(configMapName string) (*api.ConfigMap, error) { @@ -122,7 +219,7 @@ func (c *k8scache) GetEndpoints(service *api.Service) (*api.Endpoints, error) { // GetTerminatingPods returns the pods that are terminating and belong // (based on the Spec.Selector) to the supplied service. -func (c *k8scache) GetTerminatingPods(service *api.Service) (pl []*api.Pod, err error) { +func (c *k8scache) GetTerminatingPods(service *api.Service, track convtypes.TrackingTarget) (pl []*api.Pod, err error) { // converting the service selector to slice of string // in order to create the full match selector var ls []string @@ -139,6 +236,8 @@ func (c *k8scache) GetTerminatingPods(service *api.Service) (pl []*api.Pod, err return nil, err } for _, p := range list { + // all pods need to be tracked despite of the terminating status + c.tracker.Track(false, track, convtypes.PodType, p.Namespace+"/"+p.Name) if isTerminatingPod(service, p) { pl = append(pl, p) } @@ -190,16 +289,18 @@ func (c *k8scache) buildSecretName(defaultNamespace, secretName string) (string, ) } -func (c *k8scache) GetTLSSecretPath(defaultNamespace, secretName string) (file convtypes.CrtFile, err error) { +func (c *k8scache) GetTLSSecretPath(defaultNamespace, secretName string, track convtypes.TrackingTarget) (file convtypes.CrtFile, err error) { namespace, name, err := c.buildSecretName(defaultNamespace, secretName) if err != nil { return file, err } sslCert, err := c.controller.GetCertificate(namespace, name) if err != nil { + c.tracker.Track(true, track, convtypes.SecretType, namespace+"/"+name) return file, err } if sslCert.PemFileName == "" { + c.tracker.Track(true, track, convtypes.SecretType, namespace+"/"+name) return file, fmt.Errorf("secret '%s/%s' does not have keys 'tls.crt' and 'tls.key'", namespace, name) } file = convtypes.CrtFile{ @@ -208,19 +309,22 @@ func (c *k8scache) GetTLSSecretPath(defaultNamespace, secretName string) (file c CommonName: sslCert.Certificate.Subject.CommonName, NotAfter: sslCert.Certificate.NotAfter, } + c.tracker.Track(false, track, convtypes.SecretType, namespace+"/"+name) return file, nil } -func (c *k8scache) GetCASecretPath(defaultNamespace, secretName string) (ca, crl convtypes.File, err error) { +func (c *k8scache) GetCASecretPath(defaultNamespace, secretName string, track convtypes.TrackingTarget) (ca, crl convtypes.File, err error) { namespace, name, err := c.buildSecretName(defaultNamespace, secretName) if err != nil { return ca, crl, err } sslCert, err := c.controller.GetCertificate(namespace, name) if err != nil { + c.tracker.Track(true, track, convtypes.SecretType, namespace+"/"+name) return ca, crl, err } if sslCert.CAFileName == "" { + c.tracker.Track(true, track, convtypes.SecretType, namespace+"/"+name) return ca, crl, fmt.Errorf("secret '%s/%s' does not have key 'ca.crt'", namespace, name) } ca = convtypes.File{ @@ -234,6 +338,7 @@ func (c *k8scache) GetCASecretPath(defaultNamespace, secretName string) (ca, crl SHA1Hash: sslCert.PemSHA, } } + c.tracker.Track(false, track, convtypes.SecretType, namespace+"/"+name) return ca, crl, nil } @@ -262,19 +367,22 @@ func (c *k8scache) GetDHSecretPath(defaultNamespace, secretName string) (file co return file, nil } -func (c *k8scache) GetSecretContent(defaultNamespace, secretName, keyName string) ([]byte, error) { +func (c *k8scache) GetSecretContent(defaultNamespace, secretName, keyName string, track convtypes.TrackingTarget) ([]byte, error) { namespace, name, err := c.buildSecretName(defaultNamespace, secretName) if err != nil { return nil, err } secret, err := c.listers.secretLister.Secrets(namespace).Get(name) if err != nil { + c.tracker.Track(true, track, convtypes.SecretType, namespace+"/"+name) return nil, err } data, found := secret.Data[keyName] if !found { + c.tracker.Track(true, track, convtypes.SecretType, namespace+"/"+name) return nil, fmt.Errorf("secret '%s/%s' does not have key '%s'", namespace, name, keyName) } + c.tracker.Track(false, track, convtypes.SecretType, namespace+"/"+name) return data, nil } @@ -423,3 +531,153 @@ func (c *k8scache) CreateOrUpdateConfigMap(cm *api.ConfigMap) (err error) { } return err } + +// implements ListerEvents +func (c *k8scache) IsValidIngress(ing *networking.Ingress) bool { + return c.controller.IsValidClass(ing) +} + +// implements ListerEvents +func (c *k8scache) IsValidConfigMap(cm *api.ConfigMap) bool { + key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) + return key == c.globalConfigMapKey || key == c.tcpConfigMapKey +} + +// implements ListerEvents +func (c *k8scache) Notify(old, cur interface{}) { + // IMPLEMENT + // maintain a list of changed objects only if partial parsing is being used + c.stateMutex.Lock() + defer c.stateMutex.Unlock() + // old != nil: has the `old` state of a changed or removed object + // cur != nil: has the `cur` state of a changed or a just created object + // old and cur == nil: cannot identify what was changed, need to start a full resync + if old != nil { + switch old.(type) { + case *networking.Ingress: + if cur == nil { + c.ingressesDel = append(c.ingressesDel, old.(*networking.Ingress)) + } + case *api.Service: + if cur == nil { + c.servicesDel = append(c.servicesDel, old.(*api.Service)) + } + case *api.Secret: + if cur == nil { + secret := old.(*api.Secret) + c.secretsDel = append(c.secretsDel, secret) + c.controller.DeleteSecret(fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)) + } + } + } + if cur != nil { + switch cur.(type) { + case *networking.Ingress: + ing := cur.(*networking.Ingress) + if old == nil { + c.ingressesAdd = append(c.ingressesAdd, ing) + } else { + c.ingressesUpd = append(c.ingressesUpd, ing) + } + case *api.Endpoints: + c.endpointsNew = append(c.endpointsNew, cur.(*api.Endpoints)) + case *api.Service: + svc := cur.(*api.Service) + if old == nil { + c.servicesAdd = append(c.servicesAdd, svc) + } else { + c.servicesUpd = append(c.servicesUpd, svc) + } + case *api.Secret: + secret := cur.(*api.Secret) + if old == nil { + c.secretsAdd = append(c.secretsAdd, secret) + } else { + c.secretsUpd = append(c.secretsUpd, secret) + } + c.controller.UpdateSecret(fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)) + case *api.ConfigMap: + cm := cur.(*api.ConfigMap) + key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) + switch key { + case c.globalConfigMapKey: + c.globalConfigMapDataNew = cm.Data + case c.tcpConfigMapKey: + c.tcpConfigMapDataNew = cm.Data + } + case *api.Pod: + c.podsNew = append(c.podsNew, cur.(*api.Pod)) + } + } + if old == nil && cur == nil { + c.needFullSync = true + } + if c.clear { + // Notify after 500ms, giving the time to receive + // all/most of the changes of a batch update + // TODO parameterize this delay + time.AfterFunc(500*time.Millisecond, func() { c.updateQueue.Notify() }) + } + c.clear = false +} + +// implements converters.types.Cache +func (c *k8scache) SwapChangedObjects() *convtypes.ChangedObjects { + c.stateMutex.Lock() + defer c.stateMutex.Unlock() + // + changed := &convtypes.ChangedObjects{ + GlobalCur: c.globalConfigMapData, + GlobalNew: c.globalConfigMapDataNew, + TCPConfigMapCur: c.tcpConfigMapData, + TCPConfigMapNew: c.tcpConfigMapDataNew, + IngressesDel: c.ingressesDel, + IngressesUpd: c.ingressesUpd, + IngressesAdd: c.ingressesAdd, + Endpoints: c.endpointsNew, + ServicesDel: c.servicesDel, + ServicesUpd: c.servicesUpd, + ServicesAdd: c.servicesAdd, + SecretsDel: c.secretsDel, + SecretsUpd: c.secretsUpd, + SecretsAdd: c.secretsAdd, + Pods: c.podsNew, + } + // + c.podsNew = nil + c.endpointsNew = nil + // + // Secrets + // + c.secretsDel = nil + c.secretsUpd = nil + c.secretsAdd = nil + // + // Ingress + // + c.ingressesDel = nil + c.ingressesUpd = nil + c.ingressesAdd = nil + // + // ConfigMaps + // + if c.globalConfigMapDataNew != nil { + c.globalConfigMapData = c.globalConfigMapDataNew + c.globalConfigMapDataNew = nil + } + if c.tcpConfigMapDataNew != nil { + c.tcpConfigMapData = c.tcpConfigMapDataNew + c.tcpConfigMapDataNew = nil + } + // + c.clear = true + c.needFullSync = false + return changed +} + +// implements converters.types.Cache +func (c *k8scache) NeedFullSync() bool { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + return c.needFullSync +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 659407c7e..c9d3a6c9f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -20,18 +20,13 @@ import ( "context" "fmt" "net/http" - "sort" "time" "github.com/golang/glog" "github.com/spf13/pflag" api "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1beta1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes/scheme" - typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/tools/record" "github.com/jcmoraisjr/haproxy-ingress/pkg/acme" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" @@ -39,6 +34,7 @@ import ( "github.com/jcmoraisjr/haproxy-ingress/pkg/common/net/ssl" configmapconverter "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/configmap" ingressconverter "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress" + "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/tracker" ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" @@ -53,6 +49,7 @@ type HAProxyController struct { logger *logger cache *k8scache metrics *metrics + tracker convtypes.Tracker stopCh chan struct{} ingressQueue utils.Queue acmeQueue utils.Queue @@ -61,8 +58,6 @@ type HAProxyController struct { controller *controller.GenericController cfg *controller.Configuration configMap *api.ConfigMap - recorder record.EventRecorder - listers *listers converterOptions *ingtypes.ConverterOptions reloadStrategy *string maxOldConfigFiles *int @@ -105,21 +100,12 @@ func (hc *HAProxyController) configController() { hc.controller.SetNewCtrl(hc) hc.logger = &logger{depth: 1} hc.metrics = createMetrics(hc.cfg.BucketsResponseTime) - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(hc.logger.Info) - watchNamespace := hc.cfg.WatchNamespace - eventBroadcaster.StartRecordingToSink(&typedv1.EventSinkImpl{ - Interface: hc.cfg.Client.CoreV1().Events(watchNamespace), - }) - hc.recorder = eventBroadcaster.NewRecorder(scheme.Scheme, api.EventSource{ - Component: "ingress-controller", - }) - hc.listers = createListers( - hc, hc.logger, hc.recorder, hc.cfg.Client, - watchNamespace, hc.cfg.ForceNamespaceIsolation, - hc.cfg.ResyncPeriod) - hc.cache = newCache(hc.cfg.Client, hc.listers, hc.controller) hc.ingressQueue = utils.NewRateLimitingQueue(hc.cfg.RateLimitUpdate, hc.syncIngress) + hc.tracker = tracker.NewTracker() + hc.cache = createCache( + hc.logger, hc.cfg.Client, hc.controller, hc.tracker, hc.ingressQueue, + hc.cfg.WatchNamespace, hc.cfg.ForceNamespaceIsolation, + hc.cfg.ResyncPeriod) var acmeSigner acme.Signer if hc.cfg.AcmeServer { electorID := fmt.Sprintf("%s-%s", hc.cfg.AcmeElectionID, hc.cfg.IngressClass) @@ -150,6 +136,7 @@ func (hc *HAProxyController) configController() { hc.converterOptions = &ingtypes.ConverterOptions{ Logger: hc.logger, Cache: hc.cache, + Tracker: hc.tracker, AnnotationPrefix: hc.cfg.AnnPrefix, DefaultBackend: hc.cfg.DefaultService, DefaultSSLFile: hc.createDefaultSSLFile(), @@ -159,7 +146,7 @@ func (hc *HAProxyController) configController() { } func (hc *HAProxyController) startServices() { - hc.listers.RunAsync(hc.stopCh) + hc.cache.RunAsync(hc.stopCh) go hc.ingressQueue.Run() if hc.cfg.StatsCollectProcPeriod.Milliseconds() > 0 { go wait.Until(func() { @@ -193,7 +180,8 @@ func (hc *HAProxyController) stopServices() { func (hc *HAProxyController) createDefaultSSLFile() (tlsFile convtypes.CrtFile) { if hc.cfg.DefaultSSLCertificate != "" { - tlsFile, err := hc.cache.GetTLSSecretPath("", hc.cfg.DefaultSSLCertificate) + // IMPLEMENT track hosts that uses this secret/crt + tlsFile, err := hc.cache.GetTLSSecretPath("", hc.cfg.DefaultSSLCertificate, convtypes.TrackingTarget{}) if err == nil { return tlsFile } @@ -258,17 +246,10 @@ func (hc *HAProxyController) Stop() error { return err } -// Notify ... -// implements ListerEvents -// implements oldcontroller.NewCtrlIntf -func (hc *HAProxyController) Notify() { - hc.ingressQueue.Notify() -} - // GetIngressList ... // implements oldcontroller.NewCtrlIntf func (hc *HAProxyController) GetIngressList() ([]*networking.Ingress, error) { - return hc.listers.ingressLister.List(labels.Everything()) + return hc.cache.GetIngressList() } // GetSecret ... @@ -277,49 +258,6 @@ func (hc *HAProxyController) GetSecret(name string) (*api.Secret, error) { return hc.cache.GetSecret(name) } -// UpdateSecret ... -// implements ListerEvents -func (hc *HAProxyController) UpdateSecret(key string) { - hc.controller.SyncSecret(key) -} - -// DeleteSecret ... -// implements ListerEvents -func (hc *HAProxyController) DeleteSecret(key string) { - hc.controller.DeleteSecret(key) - hc.ingressQueue.Notify() -} - -// AddConfigMap ... -// implements ListerEvents -func (hc *HAProxyController) AddConfigMap(cm *api.ConfigMap) { - key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) - if key == hc.cfg.ConfigMapName { - hc.logger.InfoV(2, "adding configmap %v to backend", key) - hc.configMap = cm - } -} - -// UpdateConfigMap ... -// implements ListerEvents -func (hc *HAProxyController) UpdateConfigMap(cm *api.ConfigMap) { - key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) - if key == hc.cfg.ConfigMapName { - hc.logger.InfoV(2, "updating configmap backend (%v)", key) - hc.configMap = cm - } - if key == hc.cfg.ConfigMapName || key == hc.cfg.TCPConfigMapName { - hc.recorder.Eventf(cm, api.EventTypeNormal, "UPDATE", fmt.Sprintf("ConfigMap %v", key)) - hc.ingressQueue.Notify() - } -} - -// IsValidClass ... -// implements ListerEvents -func (hc *HAProxyController) IsValidClass(ing *networking.Ingress) bool { - return hc.controller.IsValidClass(ing) -} - // Name provides the complete name of the controller func (hc *HAProxyController) Name() string { return "HAProxy Ingress Controller" @@ -381,41 +319,18 @@ func (hc *HAProxyController) syncIngress(item interface{}) { hc.updateCount++ hc.logger.Info("starting HAProxy update id=%d", hc.updateCount) timer := utils.NewTimer(hc.metrics.ControllerProcTime) - var ingress []*networking.Ingress - il, err := hc.listers.ingressLister.List(labels.Everything()) - if err != nil { - hc.logger.Error("error reading ingress list: %v", err) - return - } - for _, ing := range il { - if hc.controller.IsValidClass(ing) { - ingress = append(ingress, ing) - } - } - sort.Slice(ingress, func(i, j int) bool { - i1 := ingress[i] - i2 := ingress[j] - if i1.CreationTimestamp != i2.CreationTimestamp { - return i1.CreationTimestamp.Before(&i2.CreationTimestamp) - } - return i1.Namespace+"/"+i1.Name < i2.Namespace+"/"+i2.Name - }) - var globalConfig map[string]string - if hc.configMap != nil { - globalConfig = hc.configMap.Data - } ingConverter := ingressconverter.NewIngressConverter( hc.converterOptions, hc.instance.Config(), - globalConfig, ) - ingConverter.Sync(ingress) + ingConverter.Sync() timer.Tick("parse_ingress") // // configmap converters // if hc.cfg.TCPConfigMapName != "" { + // TODO parses only when tcpconfigmap changes tcpConfigmap, err := hc.cache.GetConfigMap(hc.cfg.TCPConfigMapName) if err == nil && tcpConfigmap != nil { tcpSvcConverter := configmapconverter.NewTCPServicesConverter( diff --git a/pkg/controller/listers.go b/pkg/controller/listers.go index 5ed98e72b..2b9a7e58f 100644 --- a/pkg/controller/listers.go +++ b/pkg/controller/listers.go @@ -39,21 +39,16 @@ import ( // ListerEvents ... type ListerEvents interface { - Notify() - // - UpdateSecret(key string) - DeleteSecret(key string) - // - AddConfigMap(cm *api.ConfigMap) - UpdateConfigMap(cm *api.ConfigMap) - // - IsValidClass(ing *networking.Ingress) bool + IsValidIngress(ing *networking.Ingress) bool + IsValidConfigMap(cm *api.ConfigMap) bool + Notify(old, cur interface{}) } type listers struct { events ListerEvents logger types.Logger recorder record.EventRecorder + running bool // ingressLister listersv1beta1.IngressLister endpointLister listersv1.EndpointsLister @@ -136,6 +131,7 @@ func (l *listers) RunAsync(stopCh <-chan struct{}) { ) if synced { l.logger.Info("cache successfully synced") + l.running = true } else { runtime.HandleError(fmt.Errorf("initial cache sync has timed out or shutdown has requested")) } @@ -147,9 +143,11 @@ func (l *listers) createIngressLister(informer informersv1beta1.IngressInformer) l.ingressInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { ing := obj.(*networking.Ingress) - if l.events.IsValidClass(ing) { - l.events.Notify() - l.recorder.Eventf(ing, api.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", ing.Namespace, ing.Name)) + if l.events.IsValidIngress(ing) { + l.events.Notify(nil, ing) + if l.running { + l.recorder.Eventf(ing, api.EventTypeNormal, "CREATE", "Ingress %s/%s", ing.Namespace, ing.Name) + } } }, UpdateFunc: func(old, cur interface{}) { @@ -158,19 +156,21 @@ func (l *listers) createIngressLister(informer informersv1beta1.IngressInformer) } oldIng := old.(*networking.Ingress) curIng := cur.(*networking.Ingress) - oldValid := l.events.IsValidClass(oldIng) - curValid := l.events.IsValidClass(curIng) + oldValid := l.events.IsValidIngress(oldIng) + curValid := l.events.IsValidIngress(curIng) if !oldValid && !curValid { return } if !oldValid && curValid { - l.recorder.Eventf(curIng, api.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name)) + l.events.Notify(nil, curIng) + l.recorder.Eventf(curIng, api.EventTypeNormal, "CREATE", "Ingress %s/%s", curIng.Namespace, curIng.Name) } else if oldValid && !curValid { - l.recorder.Eventf(curIng, api.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name)) + l.events.Notify(oldIng, nil) + l.recorder.Eventf(curIng, api.EventTypeNormal, "DELETE", "Ingress %s/%s", curIng.Namespace, curIng.Name) } else { - l.recorder.Eventf(curIng, api.EventTypeNormal, "UPDATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name)) + l.events.Notify(oldIng, curIng) + l.recorder.Eventf(curIng, api.EventTypeNormal, "UPDATE", "Ingress %s/%s", curIng.Namespace, curIng.Name) } - l.events.Notify() }, DeleteFunc: func(obj interface{}) { ing, ok := obj.(*networking.Ingress) @@ -178,18 +178,20 @@ func (l *listers) createIngressLister(informer informersv1beta1.IngressInformer) tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { l.logger.Error("couldn't get object from tombstone %#v", obj) + l.events.Notify(nil, nil) return } if ing, ok = tombstone.Obj.(*networking.Ingress); !ok { l.logger.Error("Tombstone contained object that is not an Ingress: %#v", obj) + l.events.Notify(nil, nil) return } } - if !l.events.IsValidClass(ing) { + if !l.events.IsValidIngress(ing) { return } - l.recorder.Eventf(ing, api.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", ing.Namespace, ing.Name)) - l.events.Notify() + l.recorder.Eventf(ing, api.EventTypeNormal, "DELETE", "Ingress %s/%s", ing.Namespace, ing.Name) + l.events.Notify(ing, nil) }, }) } @@ -199,17 +201,17 @@ func (l *listers) createEndpointLister(informer informersv1.EndpointsInformer) { l.endpointInformer = informer.Informer() l.endpointInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(nil, obj) }, UpdateFunc: func(old, cur interface{}) { oldEP := old.(*api.Endpoints) curEP := cur.(*api.Endpoints) if !reflect.DeepEqual(oldEP.Subsets, curEP.Subsets) { - l.events.Notify() + l.events.Notify(oldEP, curEP) } }, DeleteFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(obj, nil) }, }) } @@ -217,6 +219,31 @@ func (l *listers) createEndpointLister(informer informersv1.EndpointsInformer) { func (l *listers) createServiceLister(informer informersv1.ServiceInformer) { l.serviceLister = informer.Lister() l.serviceInformer = informer.Informer() + l.serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + l.events.Notify(nil, obj) + }, + UpdateFunc: func(old, cur interface{}) { + if !reflect.DeepEqual(old, cur) { + l.events.Notify(old, cur) + } + }, + DeleteFunc: func(obj interface{}) { + svc, ok := obj.(*api.Service) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + l.logger.Error("couldn't get object from tombstone %#v", obj) + return + } + if svc, ok = tombstone.Obj.(*api.Service); !ok { + l.logger.Error("Tombstone contained object that is not a Service: %#v", obj) + return + } + } + l.events.Notify(svc, nil) + }, + }) } func (l *listers) createSecretLister(informer informersv1.SecretInformer) { @@ -224,13 +251,11 @@ func (l *listers) createSecretLister(informer informersv1.SecretInformer) { l.secretInformer = informer.Informer() l.secretInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(nil, obj) }, UpdateFunc: func(old, cur interface{}) { if !reflect.DeepEqual(old, cur) { - sec := cur.(*api.Secret) - key := fmt.Sprintf("%v/%v", sec.Namespace, sec.Name) - l.events.UpdateSecret(key) + l.events.Notify(old, cur) } }, DeleteFunc: func(obj interface{}) { @@ -246,8 +271,7 @@ func (l *listers) createSecretLister(informer informersv1.SecretInformer) { return } } - key := fmt.Sprintf("%v/%v", sec.Namespace, sec.Name) - l.events.DeleteSecret(key) + l.events.Notify(sec, nil) }, }) } @@ -257,11 +281,15 @@ func (l *listers) createConfigMapLister(informer informersv1.ConfigMapInformer) l.configMapInformer = informer.Informer() l.configMapInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - l.events.AddConfigMap(obj.(*api.ConfigMap)) + if l.events.IsValidConfigMap(obj.(*api.ConfigMap)) { + l.events.Notify(nil, obj) + } }, UpdateFunc: func(old, cur interface{}) { if !reflect.DeepEqual(old, cur) { - l.events.UpdateConfigMap(cur.(*api.ConfigMap)) + if l.events.IsValidConfigMap(cur.(*api.ConfigMap)) { + l.events.Notify(old, cur) + } } }, }) @@ -275,11 +303,11 @@ func (l *listers) createPodLister(informer informersv1.PodInformer) { oldPod := old.(*api.Pod) curPod := cur.(*api.Pod) if oldPod.DeletionTimestamp != curPod.DeletionTimestamp { - l.events.Notify() + l.events.Notify(old, cur) } }, DeleteFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(obj, nil) }, }) } diff --git a/pkg/converters/configmap/tcpservices.go b/pkg/converters/configmap/tcpservices.go index 1f17acf6d..3f180fe59 100644 --- a/pkg/converters/configmap/tcpservices.go +++ b/pkg/converters/configmap/tcpservices.go @@ -51,6 +51,8 @@ type tcpSvcConverter struct { var regexValidTime = regexp.MustCompile(`^[0-9]+(us|ms|s|m|h|d)$`) func (c *tcpSvcConverter) Sync(tcpservices map[string]string) { + c.haproxy.TCPBackends().RemoveAll() + // map[key]value is: // - key => port to expose // - value => ::[]:[]]::check-interval: @@ -89,7 +91,7 @@ func (c *tcpSvcConverter) Sync(tcpservices map[string]string) { } var crtfile convtypes.CrtFile if svc.secretTLS != "" { - crtfile, err = c.cache.GetTLSSecretPath("", svc.secretTLS) + crtfile, err = c.cache.GetTLSSecretPath("", svc.secretTLS, convtypes.TrackingTarget{}) if err != nil { c.logger.Warn("skipping TCP service on public port %d: %v", publicport, err) continue @@ -97,7 +99,7 @@ func (c *tcpSvcConverter) Sync(tcpservices map[string]string) { } var cafile, crlfile convtypes.File if svc.secretCA != "" { - cafile, crlfile, err = c.cache.GetCASecretPath("", svc.secretCA) + cafile, crlfile, err = c.cache.GetCASecretPath("", svc.secretCA, convtypes.TrackingTarget{}) if err != nil { c.logger.Warn("skipping TCP service on public port %d: %v", publicport, err) continue @@ -116,7 +118,7 @@ func (c *tcpSvcConverter) Sync(tcpservices map[string]string) { } } servicename := fmt.Sprintf("%s_%s", service.Namespace, service.Name) - backend := c.haproxy.AcquireTCPBackend(servicename, publicport) + backend := c.haproxy.TCPBackends().Acquire(servicename, publicport) for _, addr := range addrs { backend.AddEndpoint(addr.IP, addr.Port) } diff --git a/pkg/converters/configmap/tcpservices_test.go b/pkg/converters/configmap/tcpservices_test.go index 1f4b3c4da..9e2498c86 100644 --- a/pkg/converters/configmap/tcpservices_test.go +++ b/pkg/converters/configmap/tcpservices_test.go @@ -22,6 +22,7 @@ import ( "testing" conv_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/helper_test" + "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/tracker" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" types_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/types/helper_test" @@ -296,7 +297,7 @@ func TestTCPSvcSync(t *testing.T) { c.cache.SecretCAPath = test.secretCAMock c.cache.SecretCRLPath = test.secretCRLMock NewTCPServicesConverter(c.logger, c.haproxy, c.cache).Sync(test.services) - backends := c.haproxy.TCPBackends() + backends := c.haproxy.TCPBackends().BuildSortedItems() for _, b := range backends { for _, ep := range b.Endpoints { ep.Target = "" @@ -319,10 +320,11 @@ type testConfig struct { func setup(t *testing.T) *testConfig { logger := types_helper.NewLoggerMock(t) + tracker := tracker.NewTracker() c := &testConfig{ t: t, logger: logger, - cache: conv_helper.NewCacheMock(), + cache: conv_helper.NewCacheMock(tracker), haproxy: haproxy.CreateInstance(logger, haproxy.InstanceOptions{}).Config(), } return c diff --git a/pkg/converters/helper_test/cachemock.go b/pkg/converters/helper_test/cachemock.go index 4832a5163..3c3df2925 100644 --- a/pkg/converters/helper_test/cachemock.go +++ b/pkg/converters/helper_test/cachemock.go @@ -23,6 +23,7 @@ import ( "time" api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" ) @@ -32,6 +33,9 @@ type SecretContent map[string]map[string][]byte // CacheMock ... type CacheMock struct { + tracker convtypes.Tracker + Changed *convtypes.ChangedObjects + IngList []*networking.Ingress SvcList []*api.Service EpList map[string]*api.Endpoints TermPodList map[string][]*api.Pod @@ -44,8 +48,10 @@ type CacheMock struct { } // NewCacheMock ... -func NewCacheMock() *CacheMock { +func NewCacheMock(tracker convtypes.Tracker) *CacheMock { return &CacheMock{ + tracker: tracker, + Changed: &convtypes.ChangedObjects{}, SvcList: []*api.Service{}, EpList: map[string]*api.Endpoints{}, TermPodList: map[string][]*api.Pod{}, @@ -62,6 +68,21 @@ func (c *CacheMock) buildSecretName(defaultNamespace, secretName string) string return defaultNamespace + "/" + secretName } +// GetIngress ... +func (c *CacheMock) GetIngress(ingressName string) (*networking.Ingress, error) { + for _, ing := range c.IngList { + if ing.Namespace+"/"+ing.Name == ingressName { + return ing, nil + } + } + return nil, fmt.Errorf("ingress not found: %s", ingressName) +} + +// GetIngressList ... +func (c *CacheMock) GetIngressList() ([]*networking.Ingress, error) { + return c.IngList, nil +} + // GetService ... func (c *CacheMock) GetService(serviceName string) (*api.Service, error) { sname := strings.Split(serviceName, "/") @@ -85,7 +106,7 @@ func (c *CacheMock) GetEndpoints(service *api.Service) (*api.Endpoints, error) { } // GetTerminatingPods ... -func (c *CacheMock) GetTerminatingPods(service *api.Service) ([]*api.Pod, error) { +func (c *CacheMock) GetTerminatingPods(service *api.Service, track convtypes.TrackingTarget) ([]*api.Pod, error) { serviceName := service.Namespace + "/" + service.Name if pods, found := c.TermPodList[serviceName]; found { return pods, nil @@ -102,9 +123,10 @@ func (c *CacheMock) GetPod(podName string) (*api.Pod, error) { } // GetTLSSecretPath ... -func (c *CacheMock) GetTLSSecretPath(defaultNamespace, secretName string) (convtypes.CrtFile, error) { +func (c *CacheMock) GetTLSSecretPath(defaultNamespace, secretName string, track convtypes.TrackingTarget) (convtypes.CrtFile, error) { fullname := c.buildSecretName(defaultNamespace, secretName) if path, found := c.SecretTLSPath[fullname]; found { + c.tracker.Track(false, track, convtypes.SecretType, fullname) return convtypes.CrtFile{ Filename: path, SHA1Hash: fmt.Sprintf("%x", sha1.Sum([]byte(path))), @@ -112,11 +134,12 @@ func (c *CacheMock) GetTLSSecretPath(defaultNamespace, secretName string) (convt NotAfter: time.Now().AddDate(0, 0, 30), }, nil } + c.tracker.Track(true, track, convtypes.SecretType, fullname) return convtypes.CrtFile{}, fmt.Errorf("secret not found: '%s'", fullname) } // GetCASecretPath ... -func (c *CacheMock) GetCASecretPath(defaultNamespace, secretName string) (ca, crl convtypes.File, err error) { +func (c *CacheMock) GetCASecretPath(defaultNamespace, secretName string, track convtypes.TrackingTarget) (ca, crl convtypes.File, err error) { fullname := c.buildSecretName(defaultNamespace, secretName) if path, found := c.SecretCAPath[fullname]; found { ca = convtypes.File{ @@ -124,6 +147,7 @@ func (c *CacheMock) GetCASecretPath(defaultNamespace, secretName string) (ca, cr SHA1Hash: fmt.Sprintf("%x", sha1.Sum([]byte(path))), } } else { + c.tracker.Track(true, track, convtypes.SecretType, fullname) return ca, crl, fmt.Errorf("secret not found: '%s'", fullname) } if path, found := c.SecretCRLPath[fullname]; found { @@ -132,6 +156,7 @@ func (c *CacheMock) GetCASecretPath(defaultNamespace, secretName string) (ca, cr SHA1Hash: fmt.Sprintf("%x", sha1.Sum([]byte(path))), } } + c.tracker.Track(false, track, convtypes.SecretType, fullname) return ca, crl, nil } @@ -148,13 +173,78 @@ func (c *CacheMock) GetDHSecretPath(defaultNamespace, secretName string) (convty } // GetSecretContent ... -func (c *CacheMock) GetSecretContent(defaultNamespace, secretName, keyName string) ([]byte, error) { +func (c *CacheMock) GetSecretContent(defaultNamespace, secretName, keyName string, track convtypes.TrackingTarget) ([]byte, error) { fullname := c.buildSecretName(defaultNamespace, secretName) if content, found := c.SecretContent[fullname]; found { if val, found := content[keyName]; found { + c.tracker.Track(false, track, convtypes.SecretType, fullname) return val, nil } + c.tracker.Track(true, track, convtypes.SecretType, fullname) return nil, fmt.Errorf("secret '%s' does not have file/key '%s'", fullname, keyName) } + c.tracker.Track(true, track, convtypes.SecretType, fullname) return nil, fmt.Errorf("secret not found: '%s'", fullname) } + +// SwapChangedObjects ... +func (c *CacheMock) SwapChangedObjects() *convtypes.ChangedObjects { + changed := c.Changed + c.Changed = &convtypes.ChangedObjects{ + GlobalCur: changed.GlobalNew, + TCPConfigMapCur: changed.TCPConfigMapNew, + } + // update c.IngList based on notifications + for i, ing := range c.IngList { + for _, ingUpd := range changed.IngressesUpd { + if ing.Namespace == ingUpd.Namespace && ing.Name == ingUpd.Name { + c.IngList[i] = ingUpd + } + } + for j, ingDel := range changed.IngressesDel { + if ing.Namespace == ingDel.Namespace && ing.Name == ingDel.Name { + c.IngList[i] = c.IngList[len(c.IngList)-j-1] + } + } + } + c.IngList = c.IngList[:len(c.IngList)-len(changed.IngressesDel)] + for _, ingAdd := range changed.IngressesAdd { + c.IngList = append(c.IngList, ingAdd) + } + // update c.SvcList based on notifications + for i, svc := range c.SvcList { + for _, svcUpd := range changed.ServicesUpd { + if svc.Namespace == svcUpd.Namespace && svc.Name == svcUpd.Name { + c.SvcList[i] = svcUpd + } + } + for j, svcDel := range changed.ServicesDel { + if svc.Namespace == svcDel.Namespace && svc.Name == svcDel.Name { + c.SvcList[i] = c.SvcList[len(c.SvcList)-j-1] + delete(c.EpList, svc.Namespace+"/"+svc.Name) + } + } + } + // update c.SecretList based on notification + for _, secret := range changed.SecretsDel { + delete(c.SecretTLSPath, secret.Namespace+"/"+secret.Name) + } + for _, secret := range changed.SecretsAdd { + name := secret.Namespace + "/" + secret.Name + c.SecretTLSPath[name] = "/tls/" + name + ".pem" + } + // update c.EpList based on notifications + for _, ep := range changed.Endpoints { + c.EpList[ep.Namespace+"/"+ep.Name] = ep + } + c.SvcList = c.SvcList[:len(c.SvcList)-len(changed.ServicesDel)] + for _, svcAdd := range changed.ServicesAdd { + c.SvcList = append(c.SvcList, svcAdd) + } + return changed +} + +// NeedFullSync ... +func (c *CacheMock) NeedFullSync() bool { + return false +} diff --git a/pkg/converters/ingress/annotations/backend.go b/pkg/converters/ingress/annotations/backend.go index f9db1f53f..b665d7055 100644 --- a/pkg/converters/ingress/annotations/backend.go +++ b/pkg/converters/ingress/annotations/backend.go @@ -22,9 +22,9 @@ import ( "strconv" "strings" - "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" ingutils "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/utils" + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/utils" ) @@ -91,9 +91,16 @@ func (c *updater) buildBackendAuthHTTP(d *backData) { secretName = authSecret.Source.Namespace + "/" + secretName } listName := strings.Replace(secretName, "/", "_", 1) - userlist := c.haproxy.FindUserlist(listName) + userlist := c.haproxy.Userlists().Find(listName) if userlist == nil { - userb, err := c.cache.GetSecretContent(authSecret.Source.Namespace, authSecret.Value, "auth") + userb, err := c.cache.GetSecretContent( + authSecret.Source.Namespace, + authSecret.Value, "auth", + convtypes.TrackingTarget{ + Backend: d.backend.BackendID(), + Userlist: listName, + }, + ) if err != nil { c.logger.Error("error reading basic authentication on %v: %v", authSecret.Source, err) return nil @@ -103,7 +110,7 @@ func (c *updater) buildBackendAuthHTTP(d *backData) { for _, err := range errs { c.logger.Warn("ignoring malformed usr/passwd on secret '%s', declared on %v: %v", secretName, authSecret.Source, err) } - userlist = c.haproxy.AddUserlist(listName, users) + userlist = c.haproxy.Userlists().Replace(listName, users) if len(users) == 0 { c.logger.Warn("userlist on %v for basic authentication is empty", authSecret.Source) } @@ -639,7 +646,11 @@ func (c *updater) buildBackendProtocol(d *backData) { return } if crt := d.mapper.Get(ingtypes.BackSecureCrtSecret); crt.Value != "" { - if crtFile, err := c.cache.GetTLSSecretPath(crt.Source.Namespace, crt.Value); err == nil { + if crtFile, err := c.cache.GetTLSSecretPath( + crt.Source.Namespace, + crt.Value, + convtypes.TrackingTarget{Backend: d.backend.BackendID()}, + ); err == nil { d.backend.Server.CrtFilename = crtFile.Filename d.backend.Server.CrtHash = crtFile.SHA1Hash } else { @@ -647,7 +658,11 @@ func (c *updater) buildBackendProtocol(d *backData) { } } if ca := d.mapper.Get(ingtypes.BackSecureVerifyCASecret); ca.Value != "" { - if caFile, crlFile, err := c.cache.GetCASecretPath(ca.Source.Namespace, ca.Value); err == nil { + if caFile, crlFile, err := c.cache.GetCASecretPath( + ca.Source.Namespace, + ca.Value, + convtypes.TrackingTarget{Backend: d.backend.BackendID()}, + ); err == nil { d.backend.Server.CAFilename = caFile.Filename d.backend.Server.CAHash = caFile.SHA1Hash d.backend.Server.CRLFilename = crlFile.Filename @@ -711,7 +726,7 @@ var epNamingRegex = regexp.MustCompile(`^(seq(uence)?|pod|ip)$`) func (c *updater) buildBackendServerNaming(d *backData) { // Only warning here. d.backend.EpNaming should be updated before backend.AcquireEndpoint() - naming := d.mapper.Get(types.BackBackendServerNaming) + naming := d.mapper.Get(ingtypes.BackBackendServerNaming) if !epNamingRegex.MatchString(naming.Value) { c.logger.Warn("ignoring invalid naming type '%s' on %s, using 'seq' instead", naming.Value, naming.Source) } diff --git a/pkg/converters/ingress/annotations/backend_test.go b/pkg/converters/ingress/annotations/backend_test.go index 52499ce29..2f14f5b21 100644 --- a/pkg/converters/ingress/annotations/backend_test.go +++ b/pkg/converters/ingress/annotations/backend_test.go @@ -105,7 +105,7 @@ func TestAffinity(t *testing.T) { // 8 { ann: map[string]string{ - ingtypes.BackAffinity: "cookie", + ingtypes.BackAffinity: "cookie", ingtypes.BackSessionCookieKeywords: "nocache", }, expCookie: hatypes.Cookie{Name: "INGRESSCOOKIE", Strategy: "insert", Dynamic: false, Keywords: "nocache"}, @@ -315,7 +315,7 @@ usr2::clearpwd2`)}}, c.cache.SecretContent = test.secrets d := c.createBackendMappingData("default/app", test.source, test.annDefault, test.ann, test.paths) u.buildBackendAuthHTTP(d) - userlists := u.haproxy.Userlists() + userlists := u.haproxy.Userlists().BuildSortedItems() c.compareObjects("userlists", i, userlists, test.expUserlists) if test.expConfig != nil { c.compareObjects("auth http", i, d.backend.AuthHTTP, test.expConfig) diff --git a/pkg/converters/ingress/annotations/global.go b/pkg/converters/ingress/annotations/global.go index 0cd95e840..b4635781a 100644 --- a/pkg/converters/ingress/annotations/global.go +++ b/pkg/converters/ingress/annotations/global.go @@ -23,6 +23,7 @@ import ( "time" ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/utils" ) @@ -46,10 +47,10 @@ func (c *updater) buildGlobalAcme(d *globalData) { d.acmeData.Endpoint = endpoint d.acmeData.Expiring = time.Duration(d.mapper.Get(ingtypes.GlobalAcmeExpiring).Int()) * 24 * time.Hour d.acmeData.TermsAgreed = termsAgreed - d.acme.Prefix = "/.well-known/acme-challenge/" - d.acme.Socket = "/var/run/acme.sock" - d.acme.Enabled = true - d.acme.Shared = d.mapper.Get(ingtypes.GlobalAcmeShared).Bool() + d.global.Acme.Prefix = "/.well-known/acme-challenge/" + d.global.Acme.Socket = "/var/run/acme.sock" + d.global.Acme.Enabled = true + d.global.Acme.Shared = d.mapper.Get(ingtypes.GlobalAcmeShared).Bool() } func (c *updater) buildGlobalBind(d *globalData) { @@ -146,7 +147,7 @@ func (c *updater) buildGlobalStats(d *globalData) { d.global.Stats.BindIP = d.mapper.Get(ingtypes.GlobalBindIPAddrStats).Value d.global.Stats.Port = d.mapper.Get(ingtypes.GlobalStatsPort).Int() if tlsSecret := d.mapper.Get(ingtypes.GlobalStatsSSLCert).Value; tlsSecret != "" { - if tls, err := c.cache.GetTLSSecretPath("", tlsSecret); err == nil { + if tls, err := c.cache.GetTLSSecretPath("", tlsSecret, convtypes.TrackingTarget{}); err == nil { d.global.Stats.TLSFilename = tls.Filename d.global.Stats.TLSHash = tls.SHA1Hash } else { diff --git a/pkg/converters/ingress/annotations/host.go b/pkg/converters/ingress/annotations/host.go index 4eeaadb95..09413dc93 100644 --- a/pkg/converters/ingress/annotations/host.go +++ b/pkg/converters/ingress/annotations/host.go @@ -18,6 +18,7 @@ package annotations import ( ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" ) func (c *updater) buildHostAuthTLS(d *hostData) { @@ -30,7 +31,11 @@ func (c *updater) buildHostAuthTLS(d *hostData) { return } tls := &d.host.TLS - if cafile, crlfile, err := c.cache.GetCASecretPath(tlsSecret.Source.Namespace, tlsSecret.Value); err == nil { + if cafile, crlfile, err := c.cache.GetCASecretPath( + tlsSecret.Source.Namespace, + tlsSecret.Value, + convtypes.TrackingTarget{Hostname: d.host.Hostname}, + ); err == nil { tls.CAFilename = cafile.Filename tls.CAHash = cafile.SHA1Hash tls.CRLFilename = crlfile.Filename diff --git a/pkg/converters/ingress/annotations/mapper.go b/pkg/converters/ingress/annotations/mapper.go index f29c728e1..29e86c158 100644 --- a/pkg/converters/ingress/annotations/mapper.go +++ b/pkg/converters/ingress/annotations/mapper.go @@ -311,6 +311,11 @@ func (cv *ConfigValue) Int64() int64 { return value } +// FullName ... +func (s *Source) FullName() string { + return s.Namespace + "/" + s.Name +} + // String ... func (m *Map) String() string { return fmt.Sprintf("%+v", *m) @@ -318,5 +323,5 @@ func (m *Map) String() string { // String ... func (s *Source) String() string { - return s.Type + " '" + s.Namespace + "/" + s.Name + "'" + return s.Type + " '" + s.FullName() + "'" } diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go index d9b83698b..1da933768 100644 --- a/pkg/converters/ingress/annotations/updater.go +++ b/pkg/converters/ingress/annotations/updater.go @@ -41,6 +41,7 @@ func NewUpdater(haproxy haproxy.Config, options *ingtypes.ConverterOptions) Upda haproxy: haproxy, logger: options.Logger, cache: options.Cache, + tracker: options.Tracker, fakeCA: options.FakeCAFile, } } @@ -49,12 +50,12 @@ type updater struct { haproxy haproxy.Config logger types.Logger cache convtypes.Cache + tracker convtypes.Tracker fakeCA convtypes.CrtFile } type globalData struct { acmeData *hatypes.AcmeData - acme *hatypes.Acme global *hatypes.Global mapper *Mapper } @@ -102,7 +103,6 @@ func (c *updater) splitCIDR(cidrlist *ConfigValue) []string { func (c *updater) UpdateGlobalConfig(haproxyConfig haproxy.Config, mapper *Mapper) { d := &globalData{ acmeData: haproxyConfig.AcmeData(), - acme: haproxyConfig.Acme(), global: haproxyConfig.Global(), mapper: mapper, } diff --git a/pkg/converters/ingress/annotations/updater_test.go b/pkg/converters/ingress/annotations/updater_test.go index 9950a0ed1..42fecd7b2 100644 --- a/pkg/converters/ingress/annotations/updater_test.go +++ b/pkg/converters/ingress/annotations/updater_test.go @@ -23,6 +23,7 @@ import ( "testing" conv_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/helper_test" + "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/tracker" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" @@ -44,15 +45,18 @@ type testConfig struct { t *testing.T haproxy haproxy.Config cache *conv_helper.CacheMock + tracker convtypes.Tracker logger *types_helper.LoggerMock } func setup(t *testing.T) *testConfig { logger := &types_helper.LoggerMock{T: t} + tracker := tracker.NewTracker() return &testConfig{ t: t, haproxy: haproxy.CreateInstance(logger, haproxy.InstanceOptions{}).Config(), - cache: &conv_helper.CacheMock{}, + cache: conv_helper.NewCacheMock(tracker), + tracker: tracker, logger: logger, } } @@ -66,6 +70,7 @@ func (c *testConfig) createUpdater() *updater { haproxy: c.haproxy, cache: c.cache, logger: c.logger, + tracker: c.tracker, fakeCA: convtypes.CrtFile{ Filename: fakeCAFilename, SHA1Hash: fakeCAHash, diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index 198e7b499..f36b1e474 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -18,6 +18,8 @@ package ingress import ( "fmt" + "reflect" + "sort" "strconv" "strings" @@ -35,57 +37,225 @@ import ( // Config ... type Config interface { - Sync(ingress []*networking.Ingress) + Sync() } // NewIngressConverter ... -func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Config, globalConfig map[string]string) Config { +func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Config) Config { if options.DefaultConfig == nil { options.DefaultConfig = createDefaults } + changed := options.Cache.SwapChangedObjects() + // IMPLEMENT + // config option to allow partial parsing + // cache also need to know if partial parsing is enabled + needFullSync := options.Cache.NeedFullSync() || globalConfigNeedFullSync(changed) + globalConfig := changed.GlobalCur + if changed.GlobalNew != nil { + globalConfig = changed.GlobalNew + } defaultConfig := options.DefaultConfig() for key, value := range globalConfig { defaultConfig[key] = value } - c := &converter{ + return &converter{ haproxy: haproxy, options: options, + changed: changed, logger: options.Logger, cache: options.Cache, + tracker: options.Tracker, mapBuilder: annotations.NewMapBuilder(options.Logger, options.AnnotationPrefix+"/", defaultConfig), updater: annotations.NewUpdater(haproxy, options), globalConfig: annotations.NewMapBuilder(options.Logger, "", defaultConfig).NewMapper(), hostAnnotations: map[*hatypes.Host]*annotations.Mapper{}, backendAnnotations: map[*hatypes.Backend]*annotations.Mapper{}, + needFullSync: needFullSync, } - haproxy.ConfigDefaultX509Cert(options.DefaultSSLFile.Filename) - if options.DefaultBackend != "" { - if backend, err := c.addBackend(&annotations.Source{}, "*/", options.DefaultBackend, "", map[string]string{}); err == nil { - haproxy.Backends().SetDefaultBackend(backend) - } else { - c.logger.Error("error reading default service: %v", err) - } - } - return c } type converter struct { haproxy haproxy.Config options *ingtypes.ConverterOptions + changed *convtypes.ChangedObjects logger types.Logger cache convtypes.Cache + tracker convtypes.Tracker mapBuilder *annotations.MapBuilder updater annotations.Updater globalConfig *annotations.Mapper hostAnnotations map[*hatypes.Host]*annotations.Mapper backendAnnotations map[*hatypes.Backend]*annotations.Mapper + needFullSync bool +} + +func (c *converter) Sync() { + if c.needFullSync { + c.haproxy.Clear() + } + c.haproxy.Frontend().DefaultCert = c.options.DefaultSSLFile.Filename + if c.options.DefaultBackend != "" { + if backend, err := c.addBackend(&annotations.Source{}, "*", "/", c.options.DefaultBackend, "", map[string]string{}); err == nil { + c.haproxy.Backends().SetDefaultBackend(backend) + } else { + c.logger.Error("error reading default service: %v", err) + } + } + if c.needFullSync { + c.syncFull() + } else { + c.syncPartial() + } +} + +func globalConfigNeedFullSync(changed *convtypes.ChangedObjects) bool { + // Currently if a global is changed, all the ingress objects are parsed again. + // This need to be done due to: + // + // 1. Default host and backend annotations. If a default value + // changes, such default may impact any ingress object; + // 2. At the time of this writing, the following global + // configuration keys are used during annotation parsing: + // * GlobalDNSResolvers + // * GlobalDrainSupport + // * GlobalNoTLSRedirectLocations + // + // This might be improved after implement a way to guarantee that a global + // is just a haproxy global, default or frontend config. + cur, new := changed.GlobalCur, changed.GlobalNew + return new != nil && !reflect.DeepEqual(cur, new) } -func (c *converter) Sync(ingress []*networking.Ingress) { - for _, ing := range ingress { +func (c *converter) syncFull() { + ingList, err := c.cache.GetIngressList() + if err != nil { + c.logger.Error("error reading ingress list: %v", err) + return + } + sortIngress(ingList) + for _, ing := range ingList { c.syncIngress(ing) } - c.syncAnnotations() + c.fullSyncAnnotations() +} + +func (c *converter) syncPartial() { + // conventions: + // + // * del, upd, add: events from the listers + // * old, new: old state (deleted, before change) and new state (after change, added) + // * dirty: has impact due to a direct or indirect change + // + + // helper funcs + ing2names := func(ings []*networking.Ingress) []string { + inglist := make([]string, len(ings)) + for i, ing := range ings { + inglist[i] = ing.Namespace + "/" + ing.Name + } + return inglist + } + svc2names := func(services []*api.Service) []string { + serviceList := make([]string, len(services)) + for i, service := range services { + serviceList[i] = service.Namespace + "/" + service.Name + } + return serviceList + } + ep2names := func(endpoints []*api.Endpoints) []string { + epList := make([]string, len(endpoints)) + for i, ep := range endpoints { + epList[i] = ep.Namespace + "/" + ep.Name + } + return epList + } + secret2names := func(secrets []*api.Secret) []string { + secretList := make([]string, len(secrets)) + for i, secret := range secrets { + secretList[i] = secret.Namespace + "/" + secret.Name + } + return secretList + } + pod2names := func(pods []*api.Pod) []string { + podList := make([]string, len(pods)) + for i, pod := range pods { + podList[i] = pod.Namespace + "/" + pod.Name + } + return podList + } + + // remove changed/deleted data + delIngNames := ing2names(c.changed.IngressesDel) + updIngNames := ing2names(c.changed.IngressesUpd) + oldIngNames := append(delIngNames, updIngNames...) + delSvcNames := svc2names(c.changed.ServicesDel) + updSvcNames := svc2names(c.changed.ServicesUpd) + addSvcNames := svc2names(c.changed.ServicesAdd) + oldSvcNames := append(delSvcNames, updSvcNames...) + updEndpointsNames := ep2names(c.changed.Endpoints) + oldSvcNames = append(oldSvcNames, updEndpointsNames...) + delSecretNames := secret2names(c.changed.SecretsDel) + updSecretNames := secret2names(c.changed.SecretsUpd) + addSecretNames := secret2names(c.changed.SecretsAdd) + oldSecretNames := append(delSecretNames, updSecretNames...) + addPodNames := pod2names(c.changed.Pods) + dirtyIngs, dirtyHosts, dirtyBacks, dirtyUsers, dirtyStorages := + c.tracker.GetDirtyLinks(oldIngNames, oldSvcNames, addSvcNames, oldSecretNames, addSecretNames, addPodNames) + c.tracker.DeleteHostnames(dirtyHosts) + c.tracker.DeleteBackends(dirtyBacks) + c.tracker.DeleteUserlists(dirtyUsers) + c.tracker.DeleteStorages(dirtyStorages) + c.haproxy.Hosts().RemoveAll(dirtyHosts) + c.haproxy.Backends().RemoveAll(dirtyBacks) + c.haproxy.Userlists().RemoveAll(dirtyUsers) + c.haproxy.AcmeData().Storages().RemoveAll(dirtyStorages) + if len(dirtyHosts) > 0 || len(dirtyBacks) > 0 { + c.logger.InfoV(2, "changed hosts: %v; backends: %v", dirtyHosts, dirtyBacks) + } + + // merge dirty and added ingress objects into a single list + ingMap := make(map[string]*networking.Ingress) + for _, ing := range dirtyIngs { + ingMap[ing] = nil + } + for _, ing := range delIngNames { + delete(ingMap, ing) + } + for _, ing := range c.changed.IngressesAdd { + ingMap[ing.Namespace+"/"+ing.Name] = ing + } + ingList := make([]*networking.Ingress, 0, len(ingMap)) + for name, ing := range ingMap { + if ing == nil { + var err error + ing, err = c.cache.GetIngress(name) + if err != nil { + c.logger.Warn("ignoring ingress '%s': %v", name, err) + ing = nil + } + } + if ing != nil { + ingList = append(ingList, ing) + } + } + + // reinclude changed/added data + sortIngress(ingList) + for _, ing := range ingList { + c.syncIngress(ing) + } + c.partialSyncAnnotations(dirtyHosts, dirtyBacks) +} + +func sortIngress(ingress []*networking.Ingress) { + sort.Slice(ingress, func(i, j int) bool { + i1 := ingress[i] + i2 := ingress[j] + if i1.CreationTimestamp != i2.CreationTimestamp { + return i1.CreationTimestamp.Before(&i2.CreationTimestamp) + } + return i1.Namespace+"/"+i1.Name < i2.Namespace+"/"+i2.Name + }) } func (c *converter) syncIngress(ing *networking.Ingress) { @@ -123,7 +293,7 @@ func (c *converter) syncIngress(ing *networking.Ingress) { } svcName, svcPort := readServiceNamePort(&path.Backend) fullSvcName := ing.Namespace + "/" + svcName - backend, err := c.addBackend(source, hostname+uri, fullSvcName, svcPort, annBack) + backend, err := c.addBackend(source, hostname, uri, fullSvcName, svcPort, annBack) if err != nil { c.logger.Warn("skipping backend config of ingress '%s': %v", fullIngName, err) continue @@ -132,7 +302,7 @@ func (c *converter) syncIngress(ing *networking.Ingress) { sslpassthrough, _ := strconv.ParseBool(annHost[ingtypes.HostSSLPassthrough]) sslpasshttpport := annHost[ingtypes.HostSSLPassthroughHTTPPort] if sslpassthrough && sslpasshttpport != "" { - if _, err := c.addBackend(source, hostname+uri, fullSvcName, sslpasshttpport, annBack); err != nil { + if _, err := c.addBackend(source, hostname, uri, fullSvcName, sslpasshttpport, annBack); err != nil { c.logger.Warn("skipping http port config of ssl-passthrough on %v: %v", source, err) } } @@ -140,7 +310,7 @@ func (c *converter) syncIngress(ing *networking.Ingress) { for _, tls := range ing.Spec.TLS { for _, tlshost := range tls.Hosts { if tlshost == hostname { - tlsPath := c.addTLS(source, tls.SecretName) + tlsPath := c.addTLS(source, tlshost, tls.SecretName) if host.TLS.TLSHash == "" { host.TLS.TLSFilename = tlsPath.Filename host.TLS.TLSHash = tlsPath.SHA1Hash @@ -170,7 +340,9 @@ func (c *converter) syncIngress(ing *networking.Ingress) { } if tlsAcme { if tls.SecretName != "" { - c.haproxy.AcmeData().AddDomains(ing.Namespace+"/"+tls.SecretName, tls.Hosts) + secretName := ing.Namespace + "/" + tls.SecretName + c.haproxy.AcmeData().Storages().Acquire(secretName).AddDomains(tls.Hosts) + c.tracker.TrackStorage(convtypes.IngressType, fullIngName, secretName) } else { c.logger.Warn("skipping cert signer of ingress '%s': missing secret name", fullIngName) } @@ -178,7 +350,7 @@ func (c *converter) syncIngress(ing *networking.Ingress) { } } -func (c *converter) syncAnnotations() { +func (c *converter) fullSyncAnnotations() { c.updater.UpdateGlobalConfig(c.haproxy, c.globalConfig) for _, host := range c.haproxy.Hosts().Items() { if ann, found := c.hostAnnotations[host]; found { @@ -192,6 +364,21 @@ func (c *converter) syncAnnotations() { } } +func (c *converter) partialSyncAnnotations(hosts []string, backends []hatypes.BackendID) { + for _, hostname := range hosts { + host := c.haproxy.Hosts().FindHost(hostname) + if ann, found := c.hostAnnotations[host]; found { + c.updater.UpdateHostConfig(host, ann) + } + } + for _, backendID := range backends { + backend := c.haproxy.Backends().FindBackendID(backendID) + if ann, found := c.backendAnnotations[backend]; found { + c.updater.UpdateBackendConfig(backend, ann) + } + } +} + func (c *converter) addDefaultHostBackend(source *annotations.Source, fullSvcName, svcPort string, annHost, annBack map[string]string) error { hostname := "*" uri := "/" @@ -200,7 +387,7 @@ func (c *converter) addDefaultHostBackend(source *annotations.Source, fullSvcNam return fmt.Errorf("path %s was already defined on default host", uri) } } - backend, err := c.addBackend(source, hostname+uri, fullSvcName, svcPort, annBack) + backend, err := c.addBackend(source, hostname, uri, fullSvcName, svcPort, annBack) if err != nil { return err } @@ -210,7 +397,9 @@ func (c *converter) addDefaultHostBackend(source *annotations.Source, fullSvcNam } func (c *converter) addHost(hostname string, source *annotations.Source, ann map[string]string) *hatypes.Host { + // TODO build a stronger tracking host := c.haproxy.Hosts().AcquireHost(hostname) + c.tracker.TrackHostname(convtypes.IngressType, source.FullName(), hostname) mapper, found := c.hostAnnotations[host] if !found { mapper = c.mapBuilder.NewMapper() @@ -223,11 +412,14 @@ func (c *converter) addHost(hostname string, source *annotations.Source, ann map return host } -func (c *converter) addBackend(source *annotations.Source, hostpath, fullSvcName, svcPort string, ann map[string]string) (*hatypes.Backend, error) { +func (c *converter) addBackend(source *annotations.Source, hostname, uri, fullSvcName, svcPort string, ann map[string]string) (*hatypes.Backend, error) { + // TODO build a stronger tracking svc, err := c.cache.GetService(fullSvcName) if err != nil { + c.tracker.TrackMissingOnHostname(convtypes.ServiceType, fullSvcName, hostname) return nil, err } + c.tracker.TrackHostname(convtypes.ServiceType, fullSvcName, hostname) ssvcName := strings.Split(fullSvcName, "/") namespace := ssvcName[0] svcName := ssvcName[1] @@ -241,6 +433,8 @@ func (c *converter) addBackend(source *annotations.Source, hostpath, fullSvcName return nil, fmt.Errorf("port not found: '%s'", svcPort) } backend := c.haproxy.Backends().AcquireBackend(namespace, svcName, port.TargetPort.String()) + c.tracker.TrackBackend(convtypes.IngressType, source.FullName(), backend.BackendID()) + hostpath := hostname + uri mapper, found := c.backendAnnotations[backend] if !found { // New backend, initialize with service annotations, giving precedence @@ -285,9 +479,13 @@ func (c *converter) addBackend(source *annotations.Source, hostpath, fullSvcName return backend, nil } -func (c *converter) addTLS(source *annotations.Source, secretName string) convtypes.CrtFile { +func (c *converter) addTLS(source *annotations.Source, hostname, secretName string) convtypes.CrtFile { if secretName != "" { - tlsFile, err := c.cache.GetTLSSecretPath(source.Namespace, secretName) + tlsFile, err := c.cache.GetTLSSecretPath( + source.Namespace, + secretName, + convtypes.TrackingTarget{Hostname: hostname}, + ) if err == nil { return tlsFile } @@ -309,7 +507,7 @@ func (c *converter) addEndpoints(svc *api.Service, svcPort *api.ServicePort, bac ep := backend.AcquireEndpoint(addr.IP, addr.Port, addr.TargetRef) ep.Weight = 0 } - pods, err := c.cache.GetTerminatingPods(svc) + pods, err := c.cache.GetTerminatingPods(svc, convtypes.TrackingTarget{Backend: backend.BackendID()}) if err != nil { return err } diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index 03055b120..67acd7f07 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -32,6 +32,7 @@ import ( conv_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/helper_test" "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/annotations" + "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/tracker" ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" @@ -286,8 +287,8 @@ func TestSyncDrainSupport(t *testing.T) { pod2 := c.createPod1("default/echo-yyyyy", "172.17.1.104", "none:8080") c.cache.TermPodList[svcName] = []*api.Pod{pod1, pod2} - c.SyncDef( - map[string]string{"drain-support": "true"}, + c.cache.Changed.GlobalNew = map[string]string{"drain-support": "true"} + c.Sync( c.createIng1("default/echo", "echo.example.com", "/", "echo:8080"), ) @@ -769,6 +770,315 @@ func TestSyncMultiNamespace(t *testing.T) { port: 8080` + defaultBackendConfig) } +func TestSyncPartial(t *testing.T) { + svcDefault := [][]string{ + {"default/echo1", "8080", "172.17.0.11"}, + {"default/echo2", "8080", "172.17.0.12"}, + } + ingDefault := [][]string{ + {"default/echo1", "echo.example.com", "/app1", "echo1:8080"}, + {"default/echo2", "echo.example.com", "/app2", "echo2:8080"}, + } + ingTLSDefault := [][]string{ + {"default/echo1", "echo.example.com", "/app1", "echo1:8080", "tls1"}, + {"default/echo2", "echo.example.com", "/app2", "echo2:8080", "default/tls1"}, + } + secTLSDefault := [][]string{ + {"default/tls1"}, + } + expFrontDefault := ` +- hostname: echo.example.com + paths: + - path: /app2 + backend: default_echo2_8080 + - path: /app1 + backend: default_echo1_8080` + expBackDefault := ` +- id: default_echo1_8080 + endpoints: + - ip: 172.17.0.11 + port: 8080 +- id: default_echo2_8080 + endpoints: + - ip: 172.17.0.12 + port: 8080` + defaultBackendConfig + explogging := `INFO-V(2) changed hosts: [echo.example.com]; backends: [default_echo1_8080 default_echo2_8080]` + + testCases := []struct { + // + svc, ing, ingtls, sec [][]string + // + svcAdd, svcUpd, svcDel [][]string + ingAdd, ingUpd, ingDel [][]string + secAdd, secUpd, secDel [][]string + // + endpoints [][]string + // + expFront string + expBack string + logging string + }{ + // 0 + { + svc: svcDefault, + ing: ingDefault, + ingAdd: [][]string{ + {"default/echo3", "echo.example.com", "/app33", "echo2:8080"}, + }, + expFront: ` +- hostname: echo.example.com + paths: + - path: /app33 + backend: default_echo2_8080 + - path: /app2 + backend: default_echo2_8080 + - path: /app1 + backend: default_echo1_8080`, + expBack: expBackDefault, + }, + // 1 + { + svc: svcDefault, + ing: ingDefault, + ingUpd: [][]string{ + {"default/echo1", "echo.example.com", "/app11", "echo1:8080"}, + }, + logging: explogging, + expFront: ` +- hostname: echo.example.com + paths: + - path: /app2 + backend: default_echo2_8080 + - path: /app11 + backend: default_echo1_8080`, + expBack: expBackDefault, + }, + // 2 + { + svc: svcDefault, + ing: ingDefault, + ingDel: [][]string{ + {"default/echo1", "echo.example.com", "/app1", "echo1:8080"}, + }, + logging: explogging, + expFront: ` +- hostname: echo.example.com + paths: + - path: /app2 + backend: default_echo2_8080`, + expBack: ` +- id: default_echo2_8080 + endpoints: + - ip: 172.17.0.12 + port: 8080` + defaultBackendConfig, + }, + // 3 + { + svc: svcDefault, + ing: ingDefault, + ingAdd: [][]string{ + {"default/echo3", "echo3.example.com", "/app33", "echo2:8080"}, + }, + ingDel: [][]string{ + {"default/echo2", "echo.example.com", "/app2", "echo2:8080"}, + }, + logging: explogging, + expFront: ` +- hostname: echo.example.com + paths: + - path: /app1 + backend: default_echo1_8080 +- hostname: echo3.example.com + paths: + - path: /app33 + backend: default_echo2_8080`, + expBack: expBackDefault, + }, + // 4 + { + svc: svcDefault, + ing: [][]string{ + {"default/echo1", "echo.example.com", "/app1", "echo1:8080"}, + {"default/echo2", "echo.example.com", "/app2", "echo2:8080"}, + {"default/echo3", "echo.example.com", "/app3", "echo3:8080"}, + }, + svcAdd: [][]string{ + {"default/echo3", "8080", "172.17.0.13"}, + }, + logging: explogging, + expFront: ` +- hostname: echo.example.com + paths: + - path: /app3 + backend: default_echo3_8080 + - path: /app2 + backend: default_echo2_8080 + - path: /app1 + backend: default_echo1_8080`, + expBack: ` +- id: default_echo1_8080 + endpoints: + - ip: 172.17.0.11 + port: 8080 +- id: default_echo2_8080 + endpoints: + - ip: 172.17.0.12 + port: 8080 +- id: default_echo3_8080 + endpoints: + - ip: 172.17.0.13 + port: 8080` + defaultBackendConfig, + }, + // 5 + { + svc: svcDefault, + ing: ingDefault, + svcUpd: [][]string{ + {"default/echo2", "8080", "172.17.0.22"}, + }, + logging: explogging, + expFront: expFrontDefault, + expBack: ` +- id: default_echo1_8080 + endpoints: + - ip: 172.17.0.11 + port: 8080 +- id: default_echo2_8080 + endpoints: + - ip: 172.17.0.22 + port: 8080` + defaultBackendConfig, + }, + // 6 + { + svc: svcDefault, + ing: ingDefault, + svcDel: [][]string{ + {"default/echo2", "8080", "172.17.0.12"}, + }, + logging: explogging + ` +WARN skipping backend config of ingress 'default/echo2': service not found: 'default/echo2'`, + expFront: ` +- hostname: echo.example.com + paths: + - path: /app1 + backend: default_echo1_8080`, + expBack: ` +- id: default_echo1_8080 + endpoints: + - ip: 172.17.0.11 + port: 8080` + defaultBackendConfig, + }, + // 7 + { + svc: svcDefault, + ing: ingDefault, + endpoints: [][]string{ + {"default/echo1", "8080", "172.17.0.21,172.17.0.22,172.17.0.23"}, + }, + logging: explogging, + expFront: expFrontDefault, + expBack: ` +- id: default_echo1_8080 + endpoints: + - ip: 172.17.0.21 + port: 8080 + - ip: 172.17.0.22 + port: 8080 + - ip: 172.17.0.23 + port: 8080 +- id: default_echo2_8080 + endpoints: + - ip: 172.17.0.12 + port: 8080` + defaultBackendConfig, + }, + // 8 + { + svc: svcDefault, + ingtls: ingTLSDefault, + secAdd: secTLSDefault, + logging: explogging, + expFront: expFrontDefault + ` + tls: + tlsfilename: /tls/default/tls1.pem`, + expBack: expBackDefault, + }, + // 9 + { + svc: svcDefault, + sec: secTLSDefault, + ingtls: ingTLSDefault, + secDel: secTLSDefault, + logging: explogging + ` +WARN using default certificate due to an error reading secret 'tls1' on ingress 'default/echo1': secret not found: 'default/tls1' +WARN using default certificate due to an error reading secret 'default/tls1' on ingress 'default/echo2': secret not found: 'default/tls1'`, + expFront: expFrontDefault + ` + tls: + tlsfilename: /tls/tls-default.pem`, + expBack: expBackDefault, + }, + } + + for _, test := range testCases { + c := setup(t) + + for _, svc := range test.svc { + c.createSvc1(svc[0], svc[1], svc[2]) + } + for _, ing := range test.ing { + c.cache.IngList = append(c.cache.IngList, c.createIng1(ing[0], ing[1], ing[2], ing[3])) + } + for _, ing := range test.ingtls { + c.cache.IngList = append(c.cache.IngList, c.createIngTLS1(ing[0], ing[1], ing[2], ing[3], ing[4])) + } + for _, sec := range test.sec { + c.cache.SecretTLSPath[sec[0]] = "/tls/" + sec[0] + ".pem" + } + c.Sync() + c.logger.Logging = []string{} + + ings := func(slice *[]*networking.Ingress, params [][]string) { + for _, param := range params { + *slice = append(*slice, c.createIng1(param[0], param[1], param[2], param[3])) + } + } + svcs := func(slice *[]*api.Service, params [][]string) { + for _, param := range params { + svc, _ := c.createSvc1(param[0], param[1], param[2]) + *slice = append(*slice, svc) + } + } + secs := func(slice *[]*api.Secret, params [][]string) { + for _, param := range params { + secret := c.createSecretTLS2(param[0]) + *slice = append(*slice, secret) + } + } + endp := func(slice *[]*api.Endpoints, params [][]string) { + for _, param := range params { + _, ep := conv_helper.CreateService(param[0], param[1], param[2]) + *slice = append(*slice, ep) + } + } + ings(&c.cache.Changed.IngressesAdd, test.ingAdd) + ings(&c.cache.Changed.IngressesUpd, test.ingUpd) + ings(&c.cache.Changed.IngressesDel, test.ingDel) + svcs(&c.cache.Changed.ServicesAdd, test.svcAdd) + svcs(&c.cache.Changed.ServicesUpd, test.svcUpd) + svcs(&c.cache.Changed.ServicesDel, test.svcDel) + secs(&c.cache.Changed.SecretsAdd, test.secAdd) + secs(&c.cache.Changed.SecretsUpd, test.secUpd) + secs(&c.cache.Changed.SecretsDel, test.secDel) + endp(&c.cache.Changed.Endpoints, test.endpoints) + c.Sync() + + c.compareConfigFront(test.expFront) + c.compareConfigBack(test.expBack) + c.logger.CompareLogging(test.logging) + + c.teardown() + } +} + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * ANNOTATIONS @@ -847,7 +1157,8 @@ func TestSyncAnnFrontDefault(t *testing.T) { defer c.teardown() c.createSvc1Auto() - c.SyncDef(map[string]string{"timeout-client": "1s"}, + c.cache.Changed.GlobalNew = map[string]string{"timeout-client": "1s"} + c.Sync( c.createIng1Ann("default/echo1", "echo1.example.com", "/app", "echo:8080", map[string]string{ "ingress.kubernetes.io/timeout-client": "2s", }), @@ -967,7 +1278,8 @@ func TestSyncAnnBackDefault(t *testing.T) { c.createSvc1Ann("default/echo7", "8080", "172.17.0.17", map[string]string{ "ingress.kubernetes.io/balance-algorithm": "roundrobin", }) - c.SyncDef(map[string]string{"balance-algorithm": "roundrobin"}, + c.cache.Changed.GlobalNew = map[string]string{"balance-algorithm": "roundrobin"} + c.Sync( c.createIng1Ann("default/echo1", "echo.example.com", "/app1", "echo1:8080", map[string]string{ "ingress.kubernetes.io/balance-algorithm": "leastconn", }), @@ -1107,17 +1419,20 @@ type testConfig struct { hconfig haproxy.Config logger *types_helper.LoggerMock cache *conv_helper.CacheMock + tracker convtypes.Tracker updater *updaterMock } func setup(t *testing.T) *testConfig { logger := types_helper.NewLoggerMock(t) + tracker := tracker.NewTracker() c := &testConfig{ t: t, decode: scheme.Codecs.UniversalDeserializer().Decode, hconfig: haproxy.CreateInstance(logger, haproxy.InstanceOptions{}).Config(), - cache: conv_helper.NewCacheMock(), + cache: conv_helper.NewCacheMock(tracker), logger: logger, + tracker: tracker, } c.createSvc1("system/default", "8080", "172.17.0.99") return c @@ -1127,17 +1442,20 @@ func (c *testConfig) teardown() { c.logger.CompareLogging("") } -func (c *testConfig) Sync(ing ...*networking.Ingress) { - c.SyncDef(map[string]string{}, ing...) -} - var defaultBackendConfig = ` - id: _default_backend endpoints: - ip: 172.17.0.99 port: 8080` -func (c *testConfig) SyncDef(config map[string]string, ing ...*networking.Ingress) { +func (c *testConfig) Sync(ing ...*networking.Ingress) { + if ing != nil { + c.cache.IngList = ing + } + if c.cache.Changed.GlobalCur == nil && c.cache.Changed.GlobalNew == nil { + // first run, set GlobalNew != nil and run SyncFull + c.cache.Changed.GlobalNew = map[string]string{} + } defaultConfig := func() map[string]string { return map[string]string{ ingtypes.BackInitialWeight: "100", @@ -1147,6 +1465,7 @@ func (c *testConfig) SyncDef(config map[string]string, ing ...*networking.Ingres &ingtypes.ConverterOptions{ Cache: c.cache, Logger: c.logger, + Tracker: c.tracker, DefaultConfig: defaultConfig, DefaultBackend: "system/default", DefaultSSLFile: convtypes.CrtFile{ @@ -1158,10 +1477,9 @@ func (c *testConfig) SyncDef(config map[string]string, ing ...*networking.Ingres AnnotationPrefix: "ingress.kubernetes.io", }, c.hconfig, - config, ).(*converter) conv.updater = c.updater - conv.Sync(ing) + conv.Sync() } func (c *testConfig) createSvc1Auto() (*api.Service, *api.Endpoints) { @@ -1182,7 +1500,18 @@ func (c *testConfig) createSvc1Ann(name, port, endpoints string, ann map[string] func (c *testConfig) createSvc1(name, port, endpoints string) (*api.Service, *api.Endpoints) { svc, ep := conv_helper.CreateService(name, port, endpoints) - c.cache.SvcList = append(c.cache.SvcList, svc) + // TODO change SvcList to map + var has bool + for i, svc1 := range c.cache.SvcList { + if svc1.Namespace+"/"+svc1.Name == name { + c.cache.SvcList[i] = svc + has = true + break + } + } + if !has { + c.cache.SvcList = append(c.cache.SvcList, svc) + } c.cache.EpList[name] = ep return svc, ep } @@ -1212,6 +1541,16 @@ func (c *testConfig) createSecretTLS1(secretName string) { c.cache.SecretTLSPath[secretName] = "/tls/" + secretName + ".pem" } +func (c *testConfig) createSecretTLS2(secretName string) *api.Secret { + sname := strings.Split(secretName, "/") + return c.createObject(` +apiVersion: v1 +kind: Secret +metadata: + name: ` + sname[1] + ` + namespace: ` + sname[0]).(*api.Secret) +} + func (c *testConfig) createIng1(name, hostname, path, service string) *networking.Ingress { sname := strings.Split(name, "/") sservice := strings.Split(service, ":") @@ -1362,7 +1701,7 @@ func convertHost(hafronts ...*hatypes.Host) []hostMock { } func (c *testConfig) compareConfigFront(expected string) { - c.compareText(_yamlMarshal(convertHost(c.hconfig.Hosts().Items()...)), expected) + c.compareText(_yamlMarshal(convertHost(c.hconfig.Hosts().BuildSortedItems()...)), expected) } func (c *testConfig) compareConfigDefaultFront(expected string) { @@ -1406,5 +1745,5 @@ func convertBackend(habackends ...*hatypes.Backend) []backendMock { } func (c *testConfig) compareConfigBack(expected string) { - c.compareText(_yamlMarshal(convertBackend(c.hconfig.Backends().Items()...)), expected) + c.compareText(_yamlMarshal(convertBackend(c.hconfig.Backends().BuildSortedItems()...)), expected) } diff --git a/pkg/converters/ingress/tracker/tracker.go b/pkg/converters/ingress/tracker/tracker.go new file mode 100644 index 000000000..b83f243c3 --- /dev/null +++ b/pkg/converters/ingress/tracker/tracker.go @@ -0,0 +1,589 @@ +/* +Copyright 2020 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tracker + +import ( + "fmt" + "sort" + "strings" + + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" +) + +// NewTracker ... +func NewTracker() convtypes.Tracker { + return &tracker{} +} + +type ( + stringStringMap map[string]map[string]empty + stringBackendMap map[string]map[hatypes.BackendID]empty + backendStringMap map[hatypes.BackendID]map[string]empty + // + empty struct{} +) + +type tracker struct { + // ingress + ingressHostname stringStringMap + hostnameIngress stringStringMap + ingressBackend stringBackendMap + backendIngress backendStringMap + ingressStorages stringStringMap + storagesIngress stringStringMap + // service + serviceHostname stringStringMap + hostnameService stringStringMap + // secret + secretHostname stringStringMap + hostnameSecret stringStringMap + secretBackend stringBackendMap + backendSecret backendStringMap + secretUserlist stringStringMap + userlistSecret stringStringMap + // pod + podBackend stringBackendMap + backendPod backendStringMap + // service (missing) + serviceHostnameMissing stringStringMap + hostnameServiceMissing stringStringMap + // secret (missing) + secretHostnameMissing stringStringMap + hostnameSecretMissing stringStringMap + secretBackendMissing stringBackendMap + backendSecretMissing backendStringMap +} + +func (t *tracker) Track(isMissing bool, track convtypes.TrackingTarget, rtype convtypes.ResourceType, name string) { + if track.Hostname != "" { + if isMissing { + t.TrackMissingOnHostname(rtype, name, track.Hostname) + } else { + t.TrackHostname(rtype, name, track.Hostname) + } + } + if track.Backend.Name != "" { + if isMissing { + t.TrackMissingOnBackend(rtype, name, track.Backend) + } else { + t.TrackBackend(rtype, name, track.Backend) + } + } + if track.Userlist != "" { + if !isMissing { + t.TrackUserlist(rtype, name, track.Userlist) + } + } +} + +func (t *tracker) TrackHostname(rtype convtypes.ResourceType, name, hostname string) { + validName(name) + switch rtype { + case convtypes.IngressType: + addStringTracking(&t.ingressHostname, name, hostname) + addStringTracking(&t.hostnameIngress, hostname, name) + case convtypes.ServiceType: + addStringTracking(&t.serviceHostname, name, hostname) + addStringTracking(&t.hostnameService, hostname, name) + case convtypes.SecretType: + addStringTracking(&t.secretHostname, name, hostname) + addStringTracking(&t.hostnameSecret, hostname, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func (t *tracker) TrackBackend(rtype convtypes.ResourceType, name string, backendID hatypes.BackendID) { + validName(name) + switch rtype { + case convtypes.IngressType: + addStringBackendTracking(&t.ingressBackend, name, backendID) + addBackendStringTracking(&t.backendIngress, backendID, name) + case convtypes.SecretType: + addStringBackendTracking(&t.secretBackend, name, backendID) + addBackendStringTracking(&t.backendSecret, backendID, name) + case convtypes.PodType: + addStringBackendTracking(&t.podBackend, name, backendID) + addBackendStringTracking(&t.backendPod, backendID, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func (t *tracker) TrackUserlist(rtype convtypes.ResourceType, name, userlist string) { + validName(name) + switch rtype { + case convtypes.SecretType: + addStringTracking(&t.secretUserlist, name, userlist) + addStringTracking(&t.userlistSecret, userlist, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func (t *tracker) TrackStorage(rtype convtypes.ResourceType, name, storage string) { + validName(name) + switch rtype { + case convtypes.IngressType: + addStringTracking(&t.ingressStorages, name, storage) + addStringTracking(&t.storagesIngress, storage, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func (t *tracker) TrackMissingOnHostname(rtype convtypes.ResourceType, name, hostname string) { + validName(name) + switch rtype { + case convtypes.ServiceType: + addStringTracking(&t.serviceHostnameMissing, name, hostname) + addStringTracking(&t.hostnameServiceMissing, hostname, name) + case convtypes.SecretType: + addStringTracking(&t.secretHostnameMissing, name, hostname) + addStringTracking(&t.hostnameSecretMissing, hostname, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func (t *tracker) TrackMissingOnBackend(rtype convtypes.ResourceType, name string, backendID hatypes.BackendID) { + validName(name) + switch rtype { + case convtypes.SecretType: + addStringBackendTracking(&t.secretBackendMissing, name, backendID) + addBackendStringTracking(&t.backendSecretMissing, backendID, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func validName(name string) { + if len(strings.Split(name, "/")) != 2 { + panic(fmt.Errorf("invalid resource name: %s", name)) + } +} + +// GetDirtyLinks lists all hostnames and backendIDs that a +// list of ingress touches directly or indirectly: +// +// * when a hostname is listed, all other hostnames of all ingress that +// references it should also be listed; +// * when a backendID (service+port) is listed, all other backendIDs of +// all ingress that references it should also be listed. +// +func (t *tracker) GetDirtyLinks( + oldIngressList []string, + oldServiceList, addServiceList []string, + oldSecretList, addSecretList []string, + addPodList []string, +) (dirtyIngs, dirtyHosts []string, dirtyBacks []hatypes.BackendID, dirtyUsers, dirtyStorages []string) { + ingsMap := make(map[string]empty) + hostsMap := make(map[string]empty) + backsMap := make(map[hatypes.BackendID]empty) + usersMap := make(map[string]empty) + storagesMap := make(map[string]empty) + + // recursively fill hostsMap and backsMap from ingress and secrets + // that directly or indirectly are referenced by them + var build func([]string) + build = func(ingNames []string) { + for _, ingName := range ingNames { + ingsMap[ingName] = empty{} + for _, hostname := range t.getHostnamesByIngress(ingName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + for _, backend := range t.getBackendsByIngress(ingName) { + if _, found := backsMap[backend]; !found { + backsMap[backend] = empty{} + build(t.getIngressByBackend(backend)) + } + } + for _, storage := range t.getStoragesByIngress(ingName) { + if _, found := storagesMap[storage]; !found { + storagesMap[storage] = empty{} + build(t.getIngressByStorage(storage)) + } + } + } + } + build(oldIngressList) + // + for _, svcName := range oldServiceList { + for _, hostname := range t.getHostnamesByService(svcName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + } + for _, svcName := range addServiceList { + for _, hostname := range t.getHostnamesByServiceMissing(svcName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + } + // + for _, secretName := range oldSecretList { + for _, hostname := range t.getHostnamesBySecret(secretName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + for _, backend := range t.getBackendsBySecret(secretName) { + if _, found := backsMap[backend]; !found { + backsMap[backend] = empty{} + build(t.getIngressByBackend(backend)) + } + } + for _, userlist := range t.getUserlistsBySecret(secretName) { + if _, found := usersMap[userlist]; !found { + usersMap[userlist] = empty{} + } + } + } + for _, secretName := range addSecretList { + for _, hostname := range t.getHostnamesBySecretMissing(secretName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + for _, backend := range t.getBackendsBySecretMissing(secretName) { + if _, found := backsMap[backend]; !found { + backsMap[backend] = empty{} + build(t.getIngressByBackend(backend)) + } + } + } + // + for _, podName := range addPodList { + for _, backend := range t.getBackendsByPod(podName) { + if _, found := backsMap[backend]; !found { + backsMap[backend] = empty{} + build(t.getIngressByBackend(backend)) + } + } + } + + // convert hostsMap and backsMap to slices + if len(ingsMap) > 0 { + dirtyIngs = make([]string, 0, len(ingsMap)) + for ing := range ingsMap { + dirtyIngs = append(dirtyIngs, ing) + } + sort.Strings(dirtyIngs) + } + if len(hostsMap) > 0 { + dirtyHosts = make([]string, 0, len(hostsMap)) + for host := range hostsMap { + dirtyHosts = append(dirtyHosts, host) + } + sort.Strings(dirtyHosts) + } + if len(backsMap) > 0 { + dirtyBacks = make([]hatypes.BackendID, 0, len(backsMap)) + for back := range backsMap { + dirtyBacks = append(dirtyBacks, back) + } + sort.Slice(dirtyBacks, func(i, j int) bool { + return dirtyBacks[i].String() < dirtyBacks[j].String() + }) + } + if len(usersMap) > 0 { + dirtyUsers = make([]string, 0, len(usersMap)) + for user := range usersMap { + dirtyUsers = append(dirtyUsers, user) + } + sort.Strings(dirtyUsers) + } + if len(storagesMap) > 0 { + dirtyStorages = make([]string, 0, len(storagesMap)) + for storage := range storagesMap { + dirtyStorages = append(dirtyStorages, storage) + } + sort.Strings(dirtyStorages) + } + return dirtyIngs, dirtyHosts, dirtyBacks, dirtyUsers, dirtyStorages +} + +func (t *tracker) DeleteHostnames(hostnames []string) { + for _, hostname := range hostnames { + for ing := range t.hostnameIngress[hostname] { + deleteStringTracking(&t.ingressHostname, ing, hostname) + } + deleteStringMapKey(&t.hostnameIngress, hostname) + for service := range t.hostnameService[hostname] { + deleteStringTracking(&t.serviceHostname, service, hostname) + } + deleteStringMapKey(&t.hostnameService, hostname) + for service := range t.hostnameServiceMissing[hostname] { + deleteStringTracking(&t.serviceHostnameMissing, service, hostname) + } + deleteStringMapKey(&t.hostnameServiceMissing, hostname) + for secret := range t.hostnameSecret[hostname] { + deleteStringTracking(&t.secretHostname, secret, hostname) + } + deleteStringMapKey(&t.hostnameSecret, hostname) + for secret := range t.hostnameSecretMissing[hostname] { + deleteStringTracking(&t.secretHostnameMissing, secret, hostname) + } + deleteStringMapKey(&t.hostnameSecretMissing, hostname) + } +} + +func (t *tracker) DeleteBackends(backends []hatypes.BackendID) { + for _, backend := range backends { + for ing := range t.backendIngress[backend] { + deleteStringBackendTracking(&t.ingressBackend, ing, backend) + } + deleteBackendStringMapKey(&t.backendIngress, backend) + for secret := range t.backendSecret[backend] { + deleteStringBackendTracking(&t.secretBackend, secret, backend) + } + deleteBackendStringMapKey(&t.backendSecret, backend) + for secret := range t.backendSecretMissing[backend] { + deleteStringBackendTracking(&t.secretBackendMissing, secret, backend) + } + deleteBackendStringMapKey(&t.backendSecretMissing, backend) + for pod := range t.backendPod[backend] { + deleteStringBackendTracking(&t.podBackend, pod, backend) + } + deleteBackendStringMapKey(&t.backendPod, backend) + } +} + +func (t *tracker) DeleteUserlists(userlists []string) { + for _, userlist := range userlists { + for secret := range t.userlistSecret[userlist] { + deleteStringTracking(&t.secretUserlist, secret, userlist) + } + deleteStringMapKey(&t.userlistSecret, userlist) + } +} + +func (t *tracker) DeleteStorages(storages []string) { + for _, storage := range storages { + for ing := range t.storagesIngress[storage] { + deleteStringTracking(&t.ingressStorages, ing, storage) + } + deleteStringMapKey(&t.storagesIngress, storage) + } +} + +func (t *tracker) getIngressByHostname(hostname string) []string { + if t.hostnameIngress == nil { + return nil + } + return getStringTracking(t.hostnameIngress[hostname]) +} + +func (t *tracker) getHostnamesByIngress(ingName string) []string { + if t.ingressHostname == nil { + return nil + } + return getStringTracking(t.ingressHostname[ingName]) +} + +func (t *tracker) getIngressByBackend(backendID hatypes.BackendID) []string { + if t.backendIngress == nil { + return nil + } + return getStringTracking(t.backendIngress[backendID]) +} + +func (t *tracker) getBackendsByIngress(ingName string) []hatypes.BackendID { + if t.ingressBackend == nil { + return nil + } + return getBackendTracking(t.ingressBackend[ingName]) +} + +func (t *tracker) getIngressByStorage(storages string) []string { + if t.storagesIngress == nil { + return nil + } + return getStringTracking(t.storagesIngress[storages]) +} + +func (t *tracker) getStoragesByIngress(ingName string) []string { + if t.ingressStorages == nil { + return nil + } + return getStringTracking(t.ingressStorages[ingName]) +} + +func (t *tracker) getHostnamesByService(serviceName string) []string { + if t.serviceHostname == nil { + return nil + } + return getStringTracking(t.serviceHostname[serviceName]) +} + +func (t *tracker) getHostnamesByServiceMissing(serviceName string) []string { + if t.serviceHostnameMissing == nil { + return nil + } + return getStringTracking(t.serviceHostnameMissing[serviceName]) +} + +func (t *tracker) getHostnamesBySecret(secretName string) []string { + if t.secretHostname == nil { + return nil + } + return getStringTracking(t.secretHostname[secretName]) +} + +func (t *tracker) getHostnamesBySecretMissing(secretName string) []string { + if t.secretHostnameMissing == nil { + return nil + } + return getStringTracking(t.secretHostnameMissing[secretName]) +} + +func (t *tracker) getBackendsBySecret(secretName string) []hatypes.BackendID { + if t.secretBackend == nil { + return nil + } + return getBackendTracking(t.secretBackend[secretName]) +} + +func (t *tracker) getBackendsBySecretMissing(secretName string) []hatypes.BackendID { + if t.secretBackendMissing == nil { + return nil + } + return getBackendTracking(t.secretBackendMissing[secretName]) +} + +func (t *tracker) getUserlistsBySecret(secretName string) []string { + if t.secretUserlist == nil { + return nil + } + return getStringTracking(t.secretUserlist[secretName]) +} + +func (t *tracker) getBackendsByPod(podName string) []hatypes.BackendID { + if t.podBackend == nil { + return nil + } + return getBackendTracking(t.podBackend[podName]) +} + +func addStringTracking(trackingRef *stringStringMap, key, value string) { + if *trackingRef == nil { + *trackingRef = stringStringMap{} + } + tracking := *trackingRef + trackingMap, found := tracking[key] + if !found { + trackingMap = map[string]empty{} + tracking[key] = trackingMap + } + trackingMap[value] = empty{} +} + +func addBackendStringTracking(trackingRef *backendStringMap, key hatypes.BackendID, value string) { + if *trackingRef == nil { + *trackingRef = backendStringMap{} + } + tracking := *trackingRef + trackingMap, found := tracking[key] + if !found { + trackingMap = map[string]empty{} + tracking[key] = trackingMap + } + trackingMap[value] = empty{} +} + +func addStringBackendTracking(trackingRef *stringBackendMap, key string, value hatypes.BackendID) { + if *trackingRef == nil { + *trackingRef = stringBackendMap{} + } + tracking := *trackingRef + trackingMap, found := tracking[key] + if !found { + trackingMap = map[hatypes.BackendID]empty{} + tracking[key] = trackingMap + } + trackingMap[value] = empty{} +} + +func getStringTracking(tracking map[string]empty) []string { + stringList := make([]string, 0, len(tracking)) + for value := range tracking { + stringList = append(stringList, value) + } + return stringList +} + +func getBackendTracking(tracking map[hatypes.BackendID]empty) []hatypes.BackendID { + backendList := make([]hatypes.BackendID, 0, len(tracking)) + for value := range tracking { + backendList = append(backendList, value) + } + return backendList +} + +func deleteStringTracking(trackingRef *stringStringMap, key, value string) { + if *trackingRef == nil { + return + } + tracking := *trackingRef + trackingMap := tracking[key] + delete(trackingMap, value) + if len(trackingMap) == 0 { + delete(tracking, key) + } + if len(tracking) == 0 { + *trackingRef = nil + } +} + +func deleteStringBackendTracking(trackingRef *stringBackendMap, key string, value hatypes.BackendID) { + if *trackingRef == nil { + return + } + tracking := *trackingRef + trackingMap := tracking[key] + delete(trackingMap, value) + if len(trackingMap) == 0 { + delete(tracking, key) + } + if len(tracking) == 0 { + *trackingRef = nil + } +} + +func deleteStringMapKey(stringMap *stringStringMap, key string) { + delete(*stringMap, key) + if len(*stringMap) == 0 { + *stringMap = nil + } +} + +func deleteBackendStringMapKey(backendMap *backendStringMap, key hatypes.BackendID) { + delete(*backendMap, key) + if len(*backendMap) == 0 { + *backendMap = nil + } +} diff --git a/pkg/converters/ingress/tracker/tracker_test.go b/pkg/converters/ingress/tracker/tracker_test.go new file mode 100644 index 000000000..055c35888 --- /dev/null +++ b/pkg/converters/ingress/tracker/tracker_test.go @@ -0,0 +1,704 @@ +/* +Copyright 2020 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tracker + +import ( + "reflect" + "sort" + "testing" + + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" +) + +type hostTracking struct { + rtype convtypes.ResourceType + name string + hostname string +} + +type backTracking struct { + rtype convtypes.ResourceType + name string + backend hatypes.BackendID +} + +type userTracking struct { + rtype convtypes.ResourceType + name string + userlist string +} + +type storageTracking struct { + rtype convtypes.ResourceType + name string + storage string +} + +var ( + back1a = hatypes.BackendID{ + Namespace: "default", + Name: "svc1", + Port: "8080", + } + back1b = hatypes.BackendID{ + Namespace: "default", + Name: "svc1", + Port: "8080", + } + back2a = hatypes.BackendID{ + Namespace: "default", + Name: "svc2", + Port: "8080", + } + back2b = hatypes.BackendID{ + Namespace: "default", + Name: "svc2", + Port: "8080", + } +) + +func TestGetDirtyLinks(t *testing.T) { + testCases := []struct { + trackedHosts []hostTracking + trackedBacks []backTracking + trackedUsers []userTracking + trackedStorages []storageTracking + // + trackedMissingHosts []hostTracking + trackedMissingBacks []backTracking + // + oldIngressList []string + oldServiceList []string + addServiceList []string + oldSecretList []string + addSecretList []string + addPodList []string + // + expDirtyIngs []string + expDirtyHosts []string + expDirtyBacks []hatypes.BackendID + expDirtyUsers []string + expDirtyStorages []string + }{ + // 0 + {}, + // 1 + { + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + }, + // 2 + { + oldServiceList: []string{"default/svc1"}, + }, + // 3 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + }, + // 4 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 5 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + oldServiceList: []string{"default/svc1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 6 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + oldSecretList: []string{"default/secret1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 7 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + trackedMissingHosts: []hostTracking{ + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + addServiceList: []string{"default/svc1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 8 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + trackedMissingHosts: []hostTracking{ + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + addSecretList: []string{"default/secret1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 9 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 10 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + {convtypes.IngressType, "default/ing3", "domain2.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1", "default/ing2"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 11 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + {convtypes.IngressType, "default/ing3", "domain2.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1", "default/ing2", "default/ing3"}, + expDirtyHosts: []string{"domain1.local", "domain2.local"}, + }, + // 12 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + }, + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + expDirtyBacks: []hatypes.BackendID{back1b}, + }, + // 13 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + {convtypes.IngressType, "default/ing3", "domain3.local"}, + }, + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + {convtypes.IngressType, "default/ing2", back2a}, + {convtypes.IngressType, "default/ing3", back1b}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1", "default/ing3"}, + expDirtyHosts: []string{"domain1.local", "domain3.local"}, + expDirtyBacks: []hatypes.BackendID{back1b}, + }, + // 14 + { + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + {convtypes.SecretType, "default/secret1", back1a}, + }, + oldSecretList: []string{"default/secret1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyBacks: []hatypes.BackendID{back1b}, + }, + // 15 + { + trackedMissingBacks: []backTracking{ + {convtypes.SecretType, "default/secret1", back1a}, + }, + addSecretList: []string{"default/secret1"}, + expDirtyBacks: []hatypes.BackendID{back1b}, + }, + // 16 + { + trackedUsers: []userTracking{ + {convtypes.SecretType, "default/secret1", "usr1"}, + {convtypes.SecretType, "default/secret2", "usr2"}, + }, + oldSecretList: []string{"default/secret1"}, + expDirtyUsers: []string{"usr1"}, + }, + // 17 + { + trackedBacks: []backTracking{ + {convtypes.PodType, "default/pod1", back1a}, + {convtypes.PodType, "default/pod2", back1a}, + {convtypes.PodType, "default/pod3", back2a}, + {convtypes.PodType, "default/pod4", back2a}, + }, + addPodList: []string{"default/pod3"}, + expDirtyBacks: []hatypes.BackendID{back2b}, + }, + // 18 + { + trackedStorages: []storageTracking{ + {convtypes.IngressType, "default/ing1", "crt1"}, + {convtypes.IngressType, "default/ing2", "crt2"}, + {convtypes.IngressType, "default/ing2", "crt3"}, + {convtypes.IngressType, "default/ing3", "crt4"}, + }, + oldIngressList: []string{"default/ing2"}, + expDirtyIngs: []string{"default/ing2"}, + expDirtyStorages: []string{"crt2", "crt3"}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedHost := range test.trackedHosts { + c.tracker.TrackHostname(trackedHost.rtype, trackedHost.name, trackedHost.hostname) + } + for _, trackedBack := range test.trackedBacks { + c.tracker.TrackBackend(trackedBack.rtype, trackedBack.name, trackedBack.backend) + } + for _, trackedUser := range test.trackedUsers { + c.tracker.TrackUserlist(trackedUser.rtype, trackedUser.name, trackedUser.userlist) + } + for _, trackedStorage := range test.trackedStorages { + c.tracker.TrackStorage(trackedStorage.rtype, trackedStorage.name, trackedStorage.storage) + } + for _, trackedMissingHost := range test.trackedMissingHosts { + c.tracker.TrackMissingOnHostname(trackedMissingHost.rtype, trackedMissingHost.name, trackedMissingHost.hostname) + } + for _, trackedMissingBack := range test.trackedMissingBacks { + c.tracker.TrackMissingOnBackend(trackedMissingBack.rtype, trackedMissingBack.name, trackedMissingBack.backend) + } + dirtyIngs, dirtyHosts, dirtyBacks, dirtyUsers, dirtyStorages := + c.tracker.GetDirtyLinks( + test.oldIngressList, + test.oldServiceList, + test.addServiceList, + test.oldSecretList, + test.addSecretList, + test.addPodList, + ) + sort.Strings(dirtyIngs) + sort.Strings(dirtyHosts) + sort.Slice(dirtyBacks, func(i, j int) bool { + return dirtyBacks[i].String() < dirtyBacks[j].String() + }) + sort.Strings(dirtyUsers) + sort.Strings(dirtyStorages) + c.compareObjects("dirty ingress", i, dirtyIngs, test.expDirtyIngs) + c.compareObjects("dirty hosts", i, dirtyHosts, test.expDirtyHosts) + c.compareObjects("dirty backs", i, dirtyBacks, test.expDirtyBacks) + c.compareObjects("dirty users", i, dirtyUsers, test.expDirtyUsers) + c.compareObjects("dirty storages", i, dirtyStorages, test.expDirtyStorages) + c.teardown() + } +} + +func TestDeleteHostnames(t *testing.T) { + testCases := []struct { + trackedHosts []hostTracking + // + trackedMissingHosts []hostTracking + // + deleteHostnames []string + // + expIngressHostname stringStringMap + expHostnameIngress stringStringMap + expServiceHostname stringStringMap + expHostnameService stringStringMap + expSecretHostname stringStringMap + expHostnameSecret stringStringMap + // + expServiceHostnameMissing stringStringMap + expHostnameServiceMissing stringStringMap + expSecretHostnameMissing stringStringMap + expHostnameSecretMissing stringStringMap + }{ + // 0 + {}, + // 1 + { + deleteHostnames: []string{"domain1.local"}, + }, + // 2 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + expIngressHostname: stringStringMap{"default/ing1": {"domain1.local": empty{}}}, + expHostnameIngress: stringStringMap{"domain1.local": {"default/ing1": empty{}}}, + }, + // 3 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 4 + { + trackedHosts: []hostTracking{ + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 5 + { + trackedMissingHosts: []hostTracking{ + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 6 + { + trackedHosts: []hostTracking{ + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 7 + { + trackedMissingHosts: []hostTracking{ + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 8 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing1", "domain2.local"}, + }, + deleteHostnames: []string{"domain1.local", "domain2.local"}, + }, + // 9 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing1", "domain2.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + expIngressHostname: stringStringMap{"default/ing1": {"domain2.local": empty{}}}, + expHostnameIngress: stringStringMap{"domain2.local": {"default/ing1": empty{}}}, + }, + // 10 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 11 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local", "domain2.local"}, + }, + // 12 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing1", "domain2.local"}, + {convtypes.IngressType, "default/ing1", "domain3.local"}, + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + {convtypes.ServiceType, "default/svc1", "domain2.local"}, + {convtypes.ServiceType, "default/svc1", "domain3.local"}, + {convtypes.SecretType, "default/secret1", "domain1.local"}, + {convtypes.SecretType, "default/secret1", "domain2.local"}, + {convtypes.SecretType, "default/secret1", "domain3.local"}, + }, + deleteHostnames: []string{"domain1.local", "domain2.local"}, + expIngressHostname: stringStringMap{"default/ing1": {"domain3.local": empty{}}}, + expHostnameIngress: stringStringMap{"domain3.local": {"default/ing1": empty{}}}, + expServiceHostname: stringStringMap{"default/svc1": {"domain3.local": empty{}}}, + expHostnameService: stringStringMap{"domain3.local": {"default/svc1": empty{}}}, + expSecretHostname: stringStringMap{"default/secret1": {"domain3.local": empty{}}}, + expHostnameSecret: stringStringMap{"domain3.local": {"default/secret1": empty{}}}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedHost := range test.trackedHosts { + c.tracker.TrackHostname(trackedHost.rtype, trackedHost.name, trackedHost.hostname) + } + for _, trackedMissingHost := range test.trackedMissingHosts { + c.tracker.TrackMissingOnHostname(trackedMissingHost.rtype, trackedMissingHost.name, trackedMissingHost.hostname) + } + c.tracker.DeleteHostnames(test.deleteHostnames) + c.compareObjects("ingressHostname", i, c.tracker.ingressHostname, test.expIngressHostname) + c.compareObjects("hostnameIngress", i, c.tracker.hostnameIngress, test.expHostnameIngress) + c.compareObjects("serviceHostname", i, c.tracker.serviceHostname, test.expServiceHostname) + c.compareObjects("hostnameService", i, c.tracker.hostnameService, test.expHostnameService) + c.compareObjects("secretHostname", i, c.tracker.secretHostname, test.expSecretHostname) + c.compareObjects("hostnameSecret", i, c.tracker.hostnameSecret, test.expHostnameSecret) + c.compareObjects("serviceHostnameMissing", i, c.tracker.serviceHostnameMissing, test.expServiceHostnameMissing) + c.compareObjects("hostnameServiceMissing", i, c.tracker.hostnameServiceMissing, test.expHostnameServiceMissing) + c.compareObjects("secretHostnameMissing", i, c.tracker.secretHostnameMissing, test.expSecretHostnameMissing) + c.compareObjects("hostnameSecretMissing", i, c.tracker.hostnameSecretMissing, test.expHostnameSecretMissing) + c.teardown() + } +} + +func TestDeleteBackends(t *testing.T) { + testCases := []struct { + trackedBacks []backTracking + // + trackedMissingBacks []backTracking + // + deleteBackends []hatypes.BackendID + // + expIngressBackend stringBackendMap + expBackendIngress backendStringMap + expSecretBackend stringBackendMap + expBackendSecret backendStringMap + // + expSecretBackendMissing stringBackendMap + expBackendSecretMissing backendStringMap + }{ + // 0 + {}, + // 1 + { + deleteBackends: []hatypes.BackendID{back1b}, + }, + // 2 + { + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + }, + expBackendIngress: backendStringMap{back1b: {"default/ing1": empty{}}}, + expIngressBackend: stringBackendMap{"default/ing1": {back1b: empty{}}}, + }, + // 3 + { + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + }, + deleteBackends: []hatypes.BackendID{back1b}, + }, + // 4 + { + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + {convtypes.IngressType, "default/ing2", back1a}, + {convtypes.IngressType, "default/ing2", back2a}, + }, + deleteBackends: []hatypes.BackendID{back1b}, + expBackendIngress: backendStringMap{back2b: {"default/ing2": empty{}}}, + expIngressBackend: stringBackendMap{"default/ing2": {back2b: empty{}}}, + }, + // 5 + { + trackedBacks: []backTracking{ + {convtypes.SecretType, "default/secret1", back1a}, + {convtypes.SecretType, "default/secret2", back1a}, + {convtypes.SecretType, "default/secret2", back2a}, + }, + trackedMissingBacks: []backTracking{ + {convtypes.SecretType, "default/secret1", back1a}, + {convtypes.SecretType, "default/secret2", back1a}, + {convtypes.SecretType, "default/secret2", back2a}, + }, + deleteBackends: []hatypes.BackendID{back1b}, + expSecretBackend: stringBackendMap{"default/secret2": {back2b: empty{}}}, + expBackendSecret: backendStringMap{back2b: {"default/secret2": empty{}}}, + expSecretBackendMissing: stringBackendMap{"default/secret2": {back2b: empty{}}}, + expBackendSecretMissing: backendStringMap{back2b: {"default/secret2": empty{}}}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedBack := range test.trackedBacks { + c.tracker.TrackBackend(trackedBack.rtype, trackedBack.name, trackedBack.backend) + } + for _, trackedMissingBack := range test.trackedMissingBacks { + c.tracker.TrackMissingOnBackend(trackedMissingBack.rtype, trackedMissingBack.name, trackedMissingBack.backend) + } + c.tracker.DeleteBackends(test.deleteBackends) + c.compareObjects("ingressBackend", i, c.tracker.ingressBackend, test.expIngressBackend) + c.compareObjects("backendIngress", i, c.tracker.backendIngress, test.expBackendIngress) + c.compareObjects("secretBackend", i, c.tracker.secretBackend, test.expSecretBackend) + c.compareObjects("backendSecret", i, c.tracker.backendSecret, test.expBackendSecret) + c.compareObjects("secretBackendMissing", i, c.tracker.secretBackendMissing, test.expSecretBackendMissing) + c.compareObjects("backendSecretMissing", i, c.tracker.backendSecretMissing, test.expBackendSecretMissing) + c.teardown() + } +} + +func TestDeleteUserlists(t *testing.T) { + testCases := []struct { + trackedUsers []userTracking + // + deleteUserlists []string + // + expSecretUserlist stringStringMap + expUserlistSecret stringStringMap + }{ + // 0 + {}, + // 1 + { + deleteUserlists: []string{"usr1"}, + }, + // 2 + { + trackedUsers: []userTracking{ + {convtypes.SecretType, "default/secret1", "usr1"}, + }, + expUserlistSecret: stringStringMap{"usr1": {"default/secret1": empty{}}}, + expSecretUserlist: stringStringMap{"default/secret1": {"usr1": empty{}}}, + }, + // 3 + { + trackedUsers: []userTracking{ + {convtypes.SecretType, "default/secret1", "usr1"}, + }, + deleteUserlists: []string{"usr2"}, + expUserlistSecret: stringStringMap{"usr1": {"default/secret1": empty{}}}, + expSecretUserlist: stringStringMap{"default/secret1": {"usr1": empty{}}}, + }, + // 4 + { + trackedUsers: []userTracking{ + {convtypes.SecretType, "default/secret1", "usr1"}, + }, + deleteUserlists: []string{"usr1"}, + }, + // 5 + { + trackedUsers: []userTracking{ + {convtypes.SecretType, "default/secret1", "usr1"}, + {convtypes.SecretType, "default/secret2", "usr2"}, + }, + deleteUserlists: []string{"usr2"}, + expUserlistSecret: stringStringMap{"usr1": {"default/secret1": empty{}}}, + expSecretUserlist: stringStringMap{"default/secret1": {"usr1": empty{}}}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedUser := range test.trackedUsers { + c.tracker.TrackUserlist(trackedUser.rtype, trackedUser.name, trackedUser.userlist) + } + c.tracker.DeleteUserlists(test.deleteUserlists) + c.compareObjects("secretUserlist", i, c.tracker.secretUserlist, test.expSecretUserlist) + c.compareObjects("userlistSecret", i, c.tracker.userlistSecret, test.expUserlistSecret) + c.teardown() + } +} + +func TestDeleteStorages(t *testing.T) { + testCases := []struct { + trackedStorages []storageTracking + // + deleteStorages []string + // + expIngressStorages stringStringMap + expStoragesIngress stringStringMap + }{ + // 0 + {}, + // 1 + { + deleteStorages: []string{"crt1"}, + }, + // 2 + { + trackedStorages: []storageTracking{ + {convtypes.IngressType, "default/ingress1", "crt1"}, + }, + expIngressStorages: stringStringMap{"default/ingress1": {"crt1": empty{}}}, + expStoragesIngress: stringStringMap{"crt1": {"default/ingress1": empty{}}}, + }, + // 3 + { + trackedStorages: []storageTracking{ + {convtypes.IngressType, "default/ingress1", "crt1"}, + }, + deleteStorages: []string{"crt2"}, + expIngressStorages: stringStringMap{"default/ingress1": {"crt1": empty{}}}, + expStoragesIngress: stringStringMap{"crt1": {"default/ingress1": empty{}}}, + }, + // 4 + { + trackedStorages: []storageTracking{ + {convtypes.IngressType, "default/ingress1", "crt1"}, + }, + deleteStorages: []string{"crt1"}, + }, + // 5 + { + trackedStorages: []storageTracking{ + {convtypes.IngressType, "default/ingress1", "crt1"}, + {convtypes.IngressType, "default/ingress2", "crt2"}, + }, + deleteStorages: []string{"crt2"}, + expIngressStorages: stringStringMap{"default/ingress1": {"crt1": empty{}}}, + expStoragesIngress: stringStringMap{"crt1": {"default/ingress1": empty{}}}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedStorage := range test.trackedStorages { + c.tracker.TrackStorage(trackedStorage.rtype, trackedStorage.name, trackedStorage.storage) + } + c.tracker.DeleteStorages(test.deleteStorages) + c.compareObjects("ingressStorages", i, c.tracker.ingressStorages, test.expIngressStorages) + c.compareObjects("storagesIngress", i, c.tracker.storagesIngress, test.expStoragesIngress) + c.teardown() + } +} + +type testConfig struct { + t *testing.T + tracker *tracker +} + +func setup(t *testing.T) *testConfig { + return &testConfig{ + t: t, + tracker: NewTracker().(*tracker), + } +} + +func (c *testConfig) teardown() {} + +func (c *testConfig) compareObjects(name string, index int, actual, expected interface{}) { + if !reflect.DeepEqual(actual, expected) { + c.t.Errorf("%s on %d differs - expected: %v - actual: %v", name, index, expected, actual) + } +} diff --git a/pkg/converters/ingress/types/options.go b/pkg/converters/ingress/types/options.go index 60ee307eb..5f2a51d64 100644 --- a/pkg/converters/ingress/types/options.go +++ b/pkg/converters/ingress/types/options.go @@ -25,6 +25,7 @@ import ( type ConverterOptions struct { Logger types.Logger Cache convtypes.Cache + Tracker convtypes.Tracker DefaultConfig func() map[string]string DefaultBackend string DefaultSSLFile convtypes.CrtFile diff --git a/pkg/converters/types/interfaces.go b/pkg/converters/types/interfaces.go index 61f81a996..63847e0dc 100644 --- a/pkg/converters/types/interfaces.go +++ b/pkg/converters/types/interfaces.go @@ -20,18 +20,64 @@ import ( "time" api "k8s.io/api/core/v1" + networking "k8s.io/api/networking/v1beta1" + + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" ) // Cache ... type Cache interface { + GetIngress(ingressName string) (*networking.Ingress, error) + GetIngressList() ([]*networking.Ingress, error) GetService(serviceName string) (*api.Service, error) GetEndpoints(service *api.Service) (*api.Endpoints, error) - GetTerminatingPods(service *api.Service) ([]*api.Pod, error) + GetTerminatingPods(service *api.Service, track TrackingTarget) ([]*api.Pod, error) GetPod(podName string) (*api.Pod, error) - GetTLSSecretPath(defaultNamespace, secretName string) (CrtFile, error) - GetCASecretPath(defaultNamespace, secretName string) (ca, crl File, err error) + GetTLSSecretPath(defaultNamespace, secretName string, track TrackingTarget) (CrtFile, error) + GetCASecretPath(defaultNamespace, secretName string, track TrackingTarget) (ca, crl File, err error) GetDHSecretPath(defaultNamespace, secretName string) (File, error) - GetSecretContent(defaultNamespace, secretName, keyName string) ([]byte, error) + GetSecretContent(defaultNamespace, secretName, keyName string, track TrackingTarget) ([]byte, error) + SwapChangedObjects() *ChangedObjects + NeedFullSync() bool +} + +// ChangedObjects ... +type ChangedObjects struct { + // + GlobalCur, GlobalNew map[string]string + // + TCPConfigMapCur, TCPConfigMapNew map[string]string + // + IngressesDel, IngressesUpd, IngressesAdd []*networking.Ingress + // + Endpoints []*api.Endpoints + // + ServicesDel, ServicesUpd, ServicesAdd []*api.Service + // + SecretsDel, SecretsUpd, SecretsAdd []*api.Secret + // + Pods []*api.Pod +} + +// Tracker ... +type Tracker interface { + Track(isMissing bool, track TrackingTarget, rtype ResourceType, name string) + TrackHostname(rtype ResourceType, name, hostname string) + TrackBackend(rtype ResourceType, name string, backendID hatypes.BackendID) + TrackMissingOnHostname(rtype ResourceType, name, hostname string) + TrackStorage(rtype ResourceType, name, storage string) + GetDirtyLinks(oldIngressList, oldServiceList, addServiceList, oldSecretList, addSecretList, addPodList []string) (dirtyIngs, dirtyHosts []string, dirtyBacks []hatypes.BackendID, dirtyUsers, dirtyStorages []string) + DeleteHostnames(hostnames []string) + DeleteBackends(backends []hatypes.BackendID) + DeleteUserlists(userlists []string) + DeleteStorages(storages []string) +} + +// TrackingTarget ... +type TrackingTarget struct { + Hostname string + Backend hatypes.BackendID + Userlist string } // File ... @@ -47,3 +93,20 @@ type CrtFile struct { CommonName string NotAfter time.Time } + +// ResourceType ... +type ResourceType int + +const ( + // IngressType ... + IngressType ResourceType = iota + + // ServiceType ... + ServiceType + + // SecretType ... + SecretType + + // PodType ... + PodType +) diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 9a4b4f64e..7a6b06068 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -19,50 +19,47 @@ package haproxy import ( "fmt" "reflect" - "sort" "strings" + "github.com/jinzhu/copier" + "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/template" hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" ) // Config ... type Config interface { - AcquireTCPBackend(servicename string, port int) *hatypes.TCPBackend - ConfigDefaultX509Cert(filename string) - AddUserlist(name string, users []hatypes.User) *hatypes.Userlist - FindUserlist(name string) *hatypes.Userlist Frontend() *hatypes.Frontend SyncConfig() WriteFrontendMaps() error WriteBackendMaps() error AcmeData() *hatypes.AcmeData - Acme() *hatypes.Acme Global() *hatypes.Global - TCPBackends() []*hatypes.TCPBackend + TCPBackends() *hatypes.TCPBackends Hosts() *hatypes.Hosts Backends() *hatypes.Backends - Userlists() []*hatypes.Userlist - Equals(other Config) bool + Userlists() *hatypes.Userlists + Clear() + Commit() } type config struct { - // external state, non haproxy data, cannot reflect in Config.Equals() - acmeData *hatypes.AcmeData + // external state, non haproxy data + acmeData *hatypes.AcmeData + mapsTemplate *template.Config + mapsDir string // haproxy internal state - acme *hatypes.Acme - mapsTemplate *template.Config - mapsDir string - global *hatypes.Global - frontend *hatypes.Frontend - hosts *hatypes.Hosts - backends *hatypes.Backends - tcpbackends []*hatypes.TCPBackend - userlists []*hatypes.Userlist - defaultX509Cert string + globalOld *hatypes.Global + global *hatypes.Global + frontend *hatypes.Frontend + hosts *hatypes.Hosts + backends *hatypes.Backends + tcpbackends *hatypes.TCPBackends + userlists *hatypes.Userlists } type options struct { + // reflect changes to config.Clear() mapsTemplate *template.Config mapsDir string } @@ -74,61 +71,17 @@ func createConfig(options options) *config { } return &config{ acmeData: &hatypes.AcmeData{}, - acme: &hatypes.Acme{}, global: &hatypes.Global{}, frontend: &hatypes.Frontend{Name: "_front001"}, hosts: hatypes.CreateHosts(), backends: hatypes.CreateBackends(), + tcpbackends: hatypes.CreateTCPBackends(), + userlists: hatypes.CreateUserlists(), mapsTemplate: mapsTemplate, mapsDir: options.mapsDir, } } -func (c *config) AcquireTCPBackend(servicename string, port int) *hatypes.TCPBackend { - for _, backend := range c.tcpbackends { - if backend.Name == servicename && backend.Port == port { - return backend - } - } - backend := &hatypes.TCPBackend{ - Name: servicename, - Port: port, - } - c.tcpbackends = append(c.tcpbackends, backend) - sort.Slice(c.tcpbackends, func(i, j int) bool { - back1 := c.tcpbackends[i] - back2 := c.tcpbackends[j] - if back1.Name == back2.Name { - return back1.Port < back2.Port - } - return back1.Name < back2.Name - }) - return backend -} - -func (c *config) ConfigDefaultX509Cert(filename string) { - c.defaultX509Cert = filename -} - -func (c *config) AddUserlist(name string, users []hatypes.User) *hatypes.Userlist { - userlist := &hatypes.Userlist{ - Name: name, - Users: users, - } - sort.Slice(users, func(i, j int) bool { - return users[i].Name < users[j].Name - }) - c.userlists = append(c.userlists, userlist) - sort.Slice(c.userlists, func(i, j int) bool { - return c.userlists[i].Name < c.userlists[j].Name - }) - return userlist -} - -func (c *config) FindUserlist(name string) *hatypes.Userlist { - return nil -} - func (c *config) Frontend() *hatypes.Frontend { return c.frontend } @@ -151,7 +104,7 @@ func (c *config) SyncConfig() { c.frontend.BindSocket = c.global.Bind.HTTPSBind c.frontend.AcceptProxy = c.global.Bind.AcceptProxy } - for _, host := range c.hosts.Items() { + for _, host := range c.hosts.ItemsAdd() { if host.SSLPassthrough() { // no action if ssl-passthrough continue @@ -208,13 +161,13 @@ func (c *config) WriteFrontendMaps() error { CrtList: mapBuilder.AddMap(c.mapsDir + "/_front001_bind_crt.list"), UseServerList: mapBuilder.AddMap(c.mapsDir + "/_front001_use_server.list"), } - fmaps.CrtList.AppendItem(c.defaultX509Cert) + fmaps.CrtList.AppendItem(c.frontend.DefaultCert) // Some maps use yes/no answers instead of a list with found/missing keys // This approach avoid overlap: // 1. match with path_beg/map_beg, /path has a feature and a declared /path/sub doesn't have // 2. *.host.domain wildcard/alias/alias-regex has a feature and a declared sub.host.domain doesn't have yesno := map[bool]string{true: "yes", false: "no"} - for _, host := range c.hosts.Items() { + for _, host := range c.hosts.BuildSortedItems() { if host.SSLPassthrough() { rootPath := host.FindPath("/") if rootPath == nil { @@ -299,9 +252,9 @@ func (c *config) WriteFrontendMaps() error { tls := host.TLS crtFile := tls.TLSFilename if crtFile == "" { - crtFile = c.defaultX509Cert + crtFile = c.frontend.DefaultCert } - if crtFile != c.defaultX509Cert || tls.CAFilename != "" || tls.Ciphers != "" || tls.CipherSuites != "" { + if crtFile != c.frontend.DefaultCert || tls.CAFilename != "" || tls.Ciphers != "" || tls.CipherSuites != "" { // has custom cert, tls auth, ciphers or ciphersuites // // TODO optimization: distinct hostnames that shares crt, ca and crl @@ -375,15 +328,11 @@ func (c *config) AcmeData() *hatypes.AcmeData { return c.acmeData } -func (c *config) Acme() *hatypes.Acme { - return c.acme -} - func (c *config) Global() *hatypes.Global { return c.global } -func (c *config) TCPBackends() []*hatypes.TCPBackend { +func (c *config) TCPBackends() *hatypes.TCPBackends { return c.tcpbackends } @@ -395,17 +344,40 @@ func (c *config) Backends() *hatypes.Backends { return c.backends } -func (c *config) Userlists() []*hatypes.Userlist { +func (c *config) Userlists() *hatypes.Userlists { return c.userlists } -func (c *config) Equals(other Config) bool { - c2, ok := other.(*config) - if !ok { - return false +func (c *config) Clear() { + config := createConfig(options{ + mapsTemplate: c.mapsTemplate, + mapsDir: c.mapsDir, + }) + *c = *config +} + +func (c *config) Commit() { + if !reflect.DeepEqual(c.globalOld, c.global) { + // globals still uses the old deepCopy+fullParsing+deepEqual strategy + var globalOld hatypes.Global + if err := copier.Copy(&globalOld, c.global); err != nil { + panic(err) + } + c.globalOld = &globalOld } - // (config struct): external state, cannot reflect in Config.Equals() - copy := *c2 - copy.acmeData = c.acmeData - return reflect.DeepEqual(c, ©) + c.hosts.Commit() + c.backends.Commit() + c.tcpbackends.Commit() + c.userlists.Commit() + c.acmeData.Storages().Commit() +} + +func (c *config) hasCommittedData() bool { + // Committed data is data which was already added and synchronized + // to a haproxy instance. A `Clear()` clears the committed state. + // Whenever a commit is performed the global instance is cloned to + // its old state, and whenever a clear is performed such clone is + // cleaned as well. So a globalOld != nil is a fast and safe way to + // know if there is committed data. + return c.globalOld != nil } diff --git a/pkg/haproxy/config_test.go b/pkg/haproxy/config_test.go index 7d9046ceb..d47da07f4 100644 --- a/pkg/haproxy/config_test.go +++ b/pkg/haproxy/config_test.go @@ -59,49 +59,29 @@ func TestAcquireHostSame(t *testing.T) { } } -func TestEqual(t *testing.T) { - c1 := createConfig(options{}) - c2 := createConfig(options{}) - if !c1.Equals(c2) { - t.Error("c1 and c2 should be equals (empty)") - } - c1.ConfigDefaultX509Cert("/var/default.pem") - if c1.Equals(c2) { - t.Error("c1 and c2 should not be equals (one default cert)") - } - c2.ConfigDefaultX509Cert("/var/default.pem") - if !c1.Equals(c2) { - t.Error("c1 and c2 should be equals (default cert)") - } - b1 := c1.Backends().AcquireBackend("d", "app1", "8080") - c1.Backends().AcquireBackend("d", "app2", "8080") - if c1.Equals(c2) { - t.Error("c1 and c2 should not be equals (backends on one side)") - } - c2.Backends().AcquireBackend("d", "app2", "8080") - b2 := c2.Backends().AcquireBackend("d", "app1", "8080") - if !c1.Equals(c2) { - t.Error("c1 and c2 should be equals (with backends)") - } - h1 := c1.hosts.AcquireHost("d") - h1.AddPath(b1, "/") - if c1.Equals(c2) { - t.Error("c1 and c2 should not be equals (hosts on one side)") - } - h2 := c2.hosts.AcquireHost("d") - h2.AddPath(b2, "/") - if !c1.Equals(c2) { - t.Error("c1 and c2 should be equals (with hosts)") - } - err1 := c1.WriteFrontendMaps() - err2 := c2.WriteFrontendMaps() - if err1 != nil { - t.Errorf("error building c1: %v", err1) - } - if err2 != nil { - t.Errorf("error building c2: %v", err2) - } - if !c1.Equals(c2) { - t.Error("c1 and c2 should be equals (after building frontends)") +func TestClear(t *testing.T) { + c := createConfig(options{ + mapsDir: "/tmp/maps", + }) + c.Hosts().AcquireHost("app.local") + c.Backends().AcquireBackend("default", "app", "8080") + if c.mapsDir != "/tmp/maps" { + t.Error("expected mapsDir == /tmp/maps") + } + if len(c.Hosts().Items()) != 1 { + t.Error("expected len(hosts) == 1") + } + if len(c.Backends().Items()) != 1 { + t.Error("expected len(backends) == 1") + } + c.Clear() + if c.mapsDir != "/tmp/maps" { + t.Error("expected mapsDir == /tmp/maps") + } + if len(c.Hosts().Items()) != 0 { + t.Error("expected len(hosts) == 0") + } + if len(c.Backends().Items()) != 0 { + t.Error("expected len(backends) == 0") } } diff --git a/pkg/haproxy/dynupdate.go b/pkg/haproxy/dynupdate.go index d3dc2f225..621cc8137 100644 --- a/pkg/haproxy/dynupdate.go +++ b/pkg/haproxy/dynupdate.go @@ -30,8 +30,7 @@ import ( type dynUpdater struct { logger types.Logger - old *config - cur *config + config *config socket string cmd func(socket string, observer func(duration time.Duration), commands ...string) ([]string, error) cmdCnt int @@ -49,25 +48,17 @@ type epPair struct { } func (i *instance) newDynUpdater() *dynUpdater { - var old, cur *config - if i.oldConfig != nil { - old = i.oldConfig.(*config) - } - if i.curConfig != nil { - cur = i.curConfig.(*config) - } return &dynUpdater{ logger: i.logger, - old: old, - cur: cur, - socket: i.curConfig.Global().AdminSocket, + config: i.config.(*config), + socket: i.config.Global().AdminSocket, cmd: utils.HAProxyCommand, metrics: i.metrics, } } func (d *dynUpdater) update() bool { - updated := d.checkConfigPair() + updated := d.config.hasCommittedData() && d.checkConfigChange() if !updated { // Need to reload, time to adjust empty slots according to config d.alignSlots() @@ -75,99 +66,100 @@ func (d *dynUpdater) update() bool { return updated } -func (d *dynUpdater) checkConfigPair() bool { - oldConfig := d.old - curConfig := d.cur - if oldConfig == nil || curConfig == nil { - return false - } +func (d *dynUpdater) checkConfigChange() bool { + // updated defines if dynamic update was successfully applied. + // The udpated backend list is fully verified even if a restart + // should be made (updated=false) in order to leave haproxy as + // update as possible if a reload fails. + // TODO use two steps update and perform full dynamic update + // only if the reload failed. + updated := true - // check equality of everything but backends - oldConfigCopy := *oldConfig - oldConfigCopy.backends = curConfig.backends - if !oldConfigCopy.Equals(curConfig) { - var diff []string - if !reflect.DeepEqual(oldConfig.global, curConfig.global) { - diff = append(diff, "global") - } - if !reflect.DeepEqual(oldConfig.tcpbackends, curConfig.tcpbackends) { - diff = append(diff, "tcp-services") - } - if !reflect.DeepEqual(oldConfig.hosts, curConfig.hosts) { - diff = append(diff, "hosts") - } - if !reflect.DeepEqual(oldConfig.userlists, curConfig.userlists) { - diff = append(diff, "userlists") - } - d.logger.InfoV(2, "diff outside backends - %v", diff) - return false + var diff []string + if d.config.globalOld != nil && !reflect.DeepEqual(d.config.globalOld, d.config.global) { + diff = append(diff, "global") } - - // map backends of old and new config together - // return false if len or names doesn't match - if len(curConfig.Backends().Items()) != len(curConfig.Backends().Items()) { - d.logger.InfoV(2, "added or removed backend(s)") - return false + if d.config.tcpbackends.Changed() { + diff = append(diff, "tcp-services") + } + if d.config.hosts.Changed() { + diff = append(diff, "hosts") + } + if d.config.userlists.Changed() { + diff = append(diff, "userlists") + } + if len(diff) > 0 { + d.logger.InfoV(2, "diff outside backends: %v", diff) + updated = false } - backends := make(map[string]*backendPair, len(oldConfig.Backends().Items())) - for _, backend := range oldConfig.Backends().Items() { + + // group reusable backends together + // return false on new backend which cannot be dynamically created + backends := make(map[string]*backendPair, len(d.config.backends.ItemsDel())) + for _, backend := range d.config.backends.ItemsDel() { backends[backend.ID] = &backendPair{old: backend} } - for _, backend := range curConfig.Backends().Items() { + for _, backend := range d.config.backends.ItemsAdd() { back, found := backends[backend.ID] if !found { d.logger.InfoV(2, "added backend '%s'", backend.ID) - return false + updated = false + } else { + back.cur = backend } - back.cur = backend } // try to dynamically update every single backend // true if deep equals or sucessfully updated // false if cannot be dynamically updated or update failed for _, pair := range backends { - if !d.checkBackendPair(pair) { - return false + if pair.cur != nil && !d.checkBackendPair(pair) { + updated = false } } - return true + return updated } func (d *dynUpdater) checkBackendPair(pair *backendPair) bool { oldBack := pair.old curBack := pair.cur + // Track if dynamic update was successfully applied. + // Socket updates will continue to be applied even if updated + // is false, so haproxy will stay as updated as possible even + // if a reload fail + updated := true + // check equality of everything but endpoints oldBackCopy := *oldBack oldBackCopy.Dynamic = curBack.Dynamic oldBackCopy.Endpoints = curBack.Endpoints if !reflect.DeepEqual(&oldBackCopy, curBack) { d.logger.InfoV(2, "diff outside endpoints of backend '%s'", curBack.ID) - return false + updated = false } // can decrease endpoints, cannot increase if len(oldBack.Endpoints) < len(curBack.Endpoints) { d.logger.InfoV(2, "added endpoints on backend '%s'", curBack.ID) + // cannot continue -- missing empty slots in the backend return false } // Resolver == update via DNS discovery if curBack.Resolver != "" { - return true + return updated } - // most of the backends are equal, save some proc stopping here if deep equals - if reflect.DeepEqual(oldBack.Endpoints, curBack.Endpoints) { - return true - } - - // oldBack and curBack differs, DynUpdate is disabled, need to reload + // DynUpdate is disabled, check if differs and quit // TODO check if endpoints are the same and only the order differ if !curBack.Dynamic.DynUpdate { - d.logger.InfoV(2, "backend '%s' changed and its dynamic-scaling is 'false'", curBack.ID) - return false + if updated && !reflect.DeepEqual(oldBack.Endpoints, curBack.Endpoints) { + d.logger.InfoV(2, "backend '%s' changed and its dynamic-scaling is 'false'", curBack.ID) + return false + } + return updated } // map endpoints of old and new config together @@ -183,42 +175,42 @@ func (d *dynUpdater) checkBackendPair(pair *backendPair) bool { } } - // From this point we cannot simply `return false` because endpoint.Name - // is being updated, need to be updated until the end, and endpoints slice - // need to be sorted - updated := true - // reuse the backend/server which has the same target endpoint, if found, // this will save some socket calls and will not mess endpoint metrics var added []*hatypes.Endpoint for _, endpoint := range curBack.Endpoints { if pair, found := endpoints[endpoint.Target]; found { - endpoint.Name = pair.old.Name pair.cur = endpoint + pair.cur.Name = pair.old.Name } else { added = append(added, endpoint) } } - // try to dynamically remove/update/add endpoints - // targets used here only to have predictable results + // Try to dynamically remove/update/add endpoints. + // Targets being used here only to have predictable results (tests). // Endpoint.Label != "" means use-server of blue/green config, need reload sort.Strings(targets) for _, target := range targets { pair := endpoints[target] + if pair.cur == nil && len(added) > 0 { + pair.cur = added[0] + pair.cur.Name = pair.old.Name + added = added[1:] + } if pair.cur == nil { - if pair.old.Label != "" || (updated && !d.execDisableEndpoint(curBack.ID, pair.old)) { + if !d.execDisableEndpoint(curBack.ID, pair.old) || pair.old.Label != "" { updated = false } empty = append(empty, pair.old.Name) - } else if updated && !d.checkEndpointPair(curBack.ID, pair) { + } else if !d.checkEndpointPair(curBack.ID, pair) { updated = false } } for i := range added { // reusing empty slots from oldBack added[i].Name = empty[i] - if added[i].Label != "" || (updated && !d.execEnableEndpoint(curBack.ID, nil, added[i])) { + if !d.execEnableEndpoint(curBack.ID, nil, added[i]) || added[i].Label != "" { updated = false } } @@ -236,17 +228,15 @@ func (d *dynUpdater) checkEndpointPair(backname string, pair *epPair) bool { if reflect.DeepEqual(pair.old, pair.cur) { return true } - if pair.old.Label != "" || pair.cur.Label != "" { + updated := d.execEnableEndpoint(backname, pair.old, pair.cur) + if !updated || pair.old.Label != "" || pair.cur.Label != "" { return false } - return d.execEnableEndpoint(backname, pair.old, pair.cur) + return true } func (d *dynUpdater) alignSlots() { - if d.cur == nil { - return - } - for _, back := range d.cur.Backends().Items() { + for _, back := range d.config.Backends().Items() { if !back.Dynamic.DynUpdate { // no need to add empty slots if won't dynamically update continue diff --git a/pkg/haproxy/dynupdate_test.go b/pkg/haproxy/dynupdate_test.go index c7a596271..cdaabcdef 100644 --- a/pkg/haproxy/dynupdate_test.go +++ b/pkg/haproxy/dynupdate_test.go @@ -24,12 +24,12 @@ import ( "time" "github.com/kylelemons/godebug/diff" + + "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" ) func TestDynUpdate(t *testing.T) { testCases := []struct { - oldConfig *config - curConfig *config doconfig1 func(c *testConfig) doconfig2 func(c *testConfig) expected []string @@ -39,36 +39,89 @@ func TestDynUpdate(t *testing.T) { }{ // 0 { - oldConfig: nil, - curConfig: nil, - dynamic: false, + dynamic: true, }, // 1 { - oldConfig: nil, - curConfig: createConfig(options{}), - dynamic: false, + doconfig2: func(c *testConfig) { + c.config.Global().MaxConn = 1 + }, + dynamic: false, + logging: `INFO-V(2) diff outside backends: [global]`, }, // 2 { - oldConfig: createConfig(options{}), - curConfig: nil, - dynamic: false, + doconfig1: func(c *testConfig) { + b := c.config.Backends().AcquireBackend("default", "app", "8080") + b.AcquireEndpoint("172.17.0.2", 8080, "") + b.AcquireEndpoint("172.17.0.3", 8080, "") + }, + doconfig2: func(c *testConfig) { + b := c.config.Backends().AcquireBackend("default", "app", "8080") + b.Dynamic.DynUpdate = true + b.AcquireEndpoint("172.17.0.2", 8080, "") + b.AcquireEndpoint("172.17.0.4", 8080, "") + }, + expected: []string{ + "srv001:172.17.0.2:8080:1", + "srv002:172.17.0.4:8080:1", + }, + dynamic: true, + cmd: ` +set server default_app_8080/srv002 addr 172.17.0.4 port 8080 +set server default_app_8080/srv002 state ready +set server default_app_8080/srv002 weight 1`, + logging: `INFO-V(2) updated endpoint '172.17.0.4:8080' weight '1' state 'ready' on backend/server 'default_app_8080/srv002'`, }, // 3 { - oldConfig: createConfig(options{}), - curConfig: createConfig(options{}), - dynamic: true, + doconfig1: func(c *testConfig) { + b := c.config.Backends().AcquireBackend("default", "app", "8080") + b.AcquireEndpoint("172.17.0.2", 8080, "") + b.AcquireEndpoint("172.17.0.3", 8080, "") + b.AddEmptyEndpoint() + }, + doconfig2: func(c *testConfig) { + b := c.config.Backends().AcquireBackend("default", "app", "8080") + b.Dynamic.DynUpdate = true + b.AcquireEndpoint("172.17.0.2", 8080, "") + b.AcquireEndpoint("172.17.0.4", 8080, "") + }, + expected: []string{ + "srv001:172.17.0.2:8080:1", + "srv002:172.17.0.4:8080:1", + "srv003:127.0.0.1:1023:1", + }, + dynamic: true, + cmd: ` +set server default_app_8080/srv002 addr 172.17.0.4 port 8080 +set server default_app_8080/srv002 state ready +set server default_app_8080/srv002 weight 1`, + logging: `INFO-V(2) updated endpoint '172.17.0.4:8080' weight '1' state 'ready' on backend/server 'default_app_8080/srv002'`, }, // 4 { - oldConfig: createConfig(options{}), + doconfig1: func(c *testConfig) { + b := c.config.Backends().AcquireBackend("default", "app", "8080") + b.AcquireEndpoint("172.17.0.2", 8080, "") + b.AcquireEndpoint("172.17.0.3", 8080, "") + }, doconfig2: func(c *testConfig) { - c.config.Global().MaxConn = 1 + b := c.config.Backends().AcquireBackend("default", "app", "8080") + b.Dynamic.DynUpdate = true + b.AcquireEndpoint("172.17.0.3", 8080, "") + b.AcquireEndpoint("172.17.0.4", 8080, "") }, - dynamic: false, - logging: `INFO-V(2) diff outside backends - [global]`, + expected: []string{ + "srv001:172.17.0.4:8080:1", + "srv002:172.17.0.3:8080:1", + }, + dynamic: true, + cmd: ` +set server default_app_8080/srv001 addr 172.17.0.4 port 8080 +set server default_app_8080/srv001 state ready +set server default_app_8080/srv001 weight 1`, + logging: `INFO-V(2) updated endpoint '172.17.0.4:8080' weight '1' state 'ready' on backend/server 'default_app_8080/srv001'`, }, // 5 { @@ -377,7 +430,7 @@ set server default_app_8080/srv002 weight 1 }, dynamic: false, cmd: ``, - logging: ``, + logging: `INFO-V(2) added backend 'default_app_8080'`, }, // 16 { @@ -394,7 +447,7 @@ set server default_app_8080/srv002 weight 1 }, dynamic: false, cmd: ``, - logging: ``, + logging: `INFO-V(2) added backend 'default_app_8080'`, }, // 17 { @@ -438,6 +491,11 @@ set server default_app_8080/srv002 weight 1`, "srv002:172.17.0.4:8080:1", }, dynamic: false, + cmd: ` +set server default_app_8080/srv002 addr 172.17.0.4 port 8080 +set server default_app_8080/srv002 state ready +set server default_app_8080/srv002 weight 1`, + logging: `INFO-V(2) added endpoint '172.17.0.4:8080' weight '1' state 'ready' on backend/server 'default_app_8080/srv002'`, }, // 19 { @@ -456,6 +514,11 @@ set server default_app_8080/srv002 weight 1`, "srv002:127.0.0.1:1023:1", }, dynamic: false, + cmd: ` +set server default_app_8080/srv002 state maint +set server default_app_8080/srv002 addr 127.0.0.1 port 1023 +set server default_app_8080/srv002 weight 0`, + logging: `INFO-V(2) disabled endpoint '172.17.0.3:8080' on backend/server 'default_app_8080/srv002'`, }, // 20 { @@ -475,6 +538,11 @@ set server default_app_8080/srv002 weight 1`, "srv002:172.17.0.4:8080:1", }, dynamic: false, + cmd: ` +set server default_app_8080/srv002 addr 172.17.0.4 port 8080 +set server default_app_8080/srv002 state ready +set server default_app_8080/srv002 weight 1`, + logging: `INFO-V(2) updated endpoint '172.17.0.4:8080' weight '1' state 'ready' on backend/server 'default_app_8080/srv002'`, }, // 21 { @@ -520,19 +588,20 @@ set server default_app_8080/srv002 weight 1`, instance := c.instance.(*instance) if test.doconfig1 != nil { test.doconfig1(c) - test.oldConfig = c.config.(*config) - instance.rotateConfig() - c.config = c.newConfig() - instance.curConfig = c.config } + instance.config.Commit() + backendIDs := []types.BackendID{} + for _, backend := range c.config.Backends().Items() { + if backend != c.config.Backends().DefaultBackend() { + backendIDs = append(backendIDs, backend.BackendID()) + } + } + c.config.Backends().RemoveAll(backendIDs) if test.doconfig2 != nil { test.doconfig2(c) - test.curConfig = c.config.(*config) } var cmd string dynUpdater := instance.newDynUpdater() - dynUpdater.old = test.oldConfig - dynUpdater.cur = test.curConfig dynUpdater.cmd = func(socket string, observer func(duration time.Duration), command ...string) ([]string, error) { for _, c := range command { cmd = cmd + c + "\n" diff --git a/pkg/haproxy/instance.go b/pkg/haproxy/instance.go index ad3a7619b..9907de78a 100644 --- a/pkg/haproxy/instance.go +++ b/pkg/haproxy/instance.go @@ -19,9 +19,7 @@ package haproxy import ( "fmt" "os/exec" - "reflect" "regexp" - "sort" "strconv" "strings" @@ -69,25 +67,25 @@ func CreateInstance(logger types.Logger, options InstanceOptions) Instance { } type instance struct { + up bool logger types.Logger options *InstanceOptions templates *template.Config mapsTemplate *template.Config mapsDir string - oldConfig Config - curConfig Config + config Config metrics types.Metrics } func (i *instance) AcmeCheck(source string) (int, error) { var count int - if i.oldConfig == nil { + if !i.up { return count, fmt.Errorf("controller wasn't started yet") } if i.options.AcmeQueue == nil { return count, fmt.Errorf("Acme queue wasn't configured") } - hasAccount := i.acmeEnsureConfig(i.oldConfig.AcmeData()) + hasAccount := i.acmeEnsureConfig(i.config.AcmeData()) if !hasAccount { return count, fmt.Errorf("Cannot create or retrieve the acme client account") } @@ -98,8 +96,8 @@ func (i *instance) AcmeCheck(source string) (int, error) { return count, fmt.Errorf(msg) } i.logger.Info("starting certificate check (%s)", source) - for storage, domains := range i.oldConfig.AcmeData().Certs { - i.acmeAddCert(storage, domains) + for _, storage := range i.config.AcmeData().Storages().BuildAcmeStorages() { + i.acmeAddStorage(storage) count++ } if count == 0 { @@ -117,29 +115,17 @@ func (i *instance) acmeEnsureConfig(acmeConfig *hatypes.AcmeData) bool { return signer.HasAccount() } -func (i *instance) acmeBuildCert(storage string, domains map[string]struct{}) string { - cert := make([]string, len(domains)) - n := 0 - for dom := range domains { - cert[n] = dom - n++ - } - sort.Slice(cert, func(i, j int) bool { - return cert[i] < cert[j] - }) - return strings.Join(cert, ",") -} - -func (i *instance) acmeAddCert(storage string, domains map[string]struct{}) { - strcert := i.acmeBuildCert(storage, domains) - i.logger.InfoV(3, "enqueue certificate for processing: storage=%s domain(s)=%s", - storage, strcert) - i.options.AcmeQueue.Add(storage + "," + strcert) +func (i *instance) acmeAddStorage(storage string) { + // TODO change to a proper entity + index := strings.Index(storage, ",") + name := storage[:index] + domains := storage[index+1:] + i.logger.InfoV(3, "enqueue certificate for processing: storage=%s domain(s)=%s", name, domains) + i.options.AcmeQueue.Add(storage) } -func (i *instance) acmeRemoveCert(storage string, domains map[string]struct{}) { - strcert := i.acmeBuildCert(storage, domains) - i.options.AcmeQueue.Remove(storage + "," + strcert) +func (i *instance) acmeRemoveStorage(storage string) { + i.options.AcmeQueue.Remove(storage) } func (i *instance) ParseTemplates() error { @@ -174,23 +160,23 @@ func (i *instance) ParseTemplates() error { } func (i *instance) Config() Config { - if i.curConfig == nil { + if i.config == nil { config := createConfig(options{ mapsTemplate: i.mapsTemplate, mapsDir: i.mapsDir, }) - i.curConfig = config + i.config = config } - return i.curConfig + return i.config } var idleRegex = regexp.MustCompile(`Idle_pct: ([0-9]+)`) func (i *instance) CalcIdleMetric() { - if i.oldConfig == nil { + if !i.up { return } - msg, err := hautils.HAProxyCommand(i.oldConfig.Global().AdminSocket, i.metrics.HAProxyShowInfoResponseTime, "show info") + msg, err := hautils.HAProxyCommand(i.config.Global().AdminSocket, i.metrics.HAProxyShowInfoResponseTime, "show info") if err != nil { i.logger.Error("error reading admin socket: %v", err) return @@ -213,48 +199,30 @@ func (i *instance) Update(timer *utils.Timer) { } func (i *instance) acmeUpdate() { - if i.oldConfig == nil || i.curConfig == nil || i.options.AcmeQueue == nil { + if i.config == nil || i.options.AcmeQueue == nil { return } + storages := i.config.AcmeData().Storages() le := i.options.LeaderElector if le.IsLeader() { - hasAccount := i.acmeEnsureConfig(i.curConfig.AcmeData()) + hasAccount := i.acmeEnsureConfig(i.config.AcmeData()) if !hasAccount { return } - } - var updated bool - oldCerts := i.oldConfig.AcmeData().Certs - curCerts := i.curConfig.AcmeData().Certs - // Remove from the retry queue certs that was removed from the config - for storage, domains := range oldCerts { - curdomains, found := curCerts[storage] - if !found || !reflect.DeepEqual(domains, curdomains) { - if le.IsLeader() { - i.acmeRemoveCert(storage, domains) - } - updated = true + for _, add := range storages.BuildAcmeStoragesAdd() { + i.acmeAddStorage(add) } - } - // Add new certs to the work queue - for storage, domains := range curCerts { - olddomains, found := oldCerts[storage] - if !found || !reflect.DeepEqual(domains, olddomains) { - if le.IsLeader() { - i.acmeAddCert(storage, domains) - } - updated = true + for _, del := range storages.BuildAcmeStoragesDel() { + i.acmeRemoveStorage(del) } - } - if updated && !le.IsLeader() { + } else if storages.Updated() { i.logger.InfoV(2, "skipping acme update check, leader is %s", le.LeaderName()) } } func (i *instance) haproxyUpdate(timer *utils.Timer) { // nil config, just ignore - if i.curConfig == nil { - i.logger.Info("new configuration is empty") + if i.config == nil { return } // @@ -263,31 +231,27 @@ func (i *instance) haproxyUpdate(timer *utils.Timer) { // - i.metrics.IncUpdate() should be called always, but only once // - i.metrics.UpdateSuccessful() should be called only if haproxy is reloaded or cfg is validated // - defer i.rotateConfig() - i.curConfig.SyncConfig() - if err := i.curConfig.WriteFrontendMaps(); err != nil { - i.logger.Error("error building configuration group: %v", err) + defer i.config.Commit() + i.config.SyncConfig() + if err := i.config.WriteFrontendMaps(); err != nil { + i.logger.Error("error building frontend maps: %v", err) i.metrics.IncUpdateNoop() return } - if err := i.curConfig.WriteBackendMaps(); err != nil { + if err := i.config.WriteBackendMaps(); err != nil { i.logger.Error("error building backend maps: %v", err) i.metrics.IncUpdateNoop() return } - if i.curConfig.Equals(i.oldConfig) { - i.logger.InfoV(2, "old and new configurations match, skipping reload") - i.metrics.IncUpdateNoop() - return - } updater := i.newDynUpdater() updated := updater.update() + timer.Tick("write_maps") if !updated || updater.cmdCnt > 0 { // only need to rewrtite config files if: // - !updated - there are changes that cannot be dynamically applied // - updater.cmdCnt > 0 - there are changes that was dynamically applied - err := i.templates.Write(i.curConfig) - timer.Tick("write_tmpl") + err := i.templates.Write(i.config) + timer.Tick("write_config") if err != nil { i.logger.Error("error writing configuration: %v", err) i.metrics.IncUpdateNoop() @@ -319,37 +283,30 @@ func (i *instance) haproxyUpdate(timer *utils.Timer) { i.metrics.UpdateSuccessful(false) return } - timer.Tick("reload_haproxy") + i.up = true i.metrics.UpdateSuccessful(true) i.logger.Info("HAProxy successfully reloaded") + timer.Tick("reload_haproxy") } func (i *instance) updateCertExpiring() { // TODO move to dynupdate when dynamic crt update is implemented - if i.oldConfig == nil { - for _, curHost := range i.curConfig.Hosts().Items() { - if curHost.TLS.HasTLS() { - i.metrics.SetCertExpireDate(curHost.Hostname, curHost.TLS.TLSCommonName, &curHost.TLS.TLSNotAfter) + hostsAdd := i.config.Hosts().ItemsAdd() + hostsDel := i.config.Hosts().ItemsDel() + for hostname, oldHost := range hostsDel { + if oldHost.TLS.HasTLS() { + curHost, found := hostsAdd[hostname] + if !found || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName { + i.metrics.SetCertExpireDate(hostname, oldHost.TLS.TLSCommonName, nil) } } - return - } - for _, oldHost := range i.oldConfig.Hosts().Items() { - if !oldHost.TLS.HasTLS() { - continue - } - curHost := i.curConfig.Hosts().FindHost(oldHost.Hostname) - if curHost == nil || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName { - i.metrics.SetCertExpireDate(oldHost.Hostname, oldHost.TLS.TLSCommonName, nil) - } } - for _, curHost := range i.curConfig.Hosts().Items() { - if !curHost.TLS.HasTLS() { - continue - } - oldHost := i.oldConfig.Hosts().FindHost(curHost.Hostname) - if oldHost == nil || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName || oldHost.TLS.TLSNotAfter != curHost.TLS.TLSNotAfter { - i.metrics.SetCertExpireDate(curHost.Hostname, curHost.TLS.TLSCommonName, &curHost.TLS.TLSNotAfter) + for hostname, curHost := range hostsAdd { + if curHost.TLS.HasTLS() { + oldHost, found := hostsDel[hostname] + if !found || oldHost.TLS.TLSCommonName != curHost.TLS.TLSCommonName || oldHost.TLS.TLSNotAfter != curHost.TLS.TLSNotAfter { + i.metrics.SetCertExpireDate(hostname, curHost.TLS.TLSCommonName, &curHost.TLS.TLSNotAfter) + } } } } @@ -382,9 +339,3 @@ func (i *instance) reload() error { } return nil } - -func (i *instance) rotateConfig() { - // TODO releaseConfig (old support files, ...) - i.oldConfig = i.curConfig - i.curConfig = nil -} diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index 38d6399d2..56e3df9b8 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -1137,7 +1137,7 @@ func TestInstanceTCPBackend(t *testing.T) { // 0 { doconfig: func(c *testConfig) { - b := c.config.AcquireTCPBackend("postgresql", 5432) + b := c.config.TCPBackends().Acquire("postgresql", 5432) b.AddEndpoint("172.17.0.2", 5432) }, expected: ` @@ -1149,7 +1149,7 @@ listen _tcp_postgresql_5432 // 1 { doconfig: func(c *testConfig) { - b := c.config.AcquireTCPBackend("pq", 5432) + b := c.config.TCPBackends().Acquire("pq", 5432) b.AddEndpoint("172.17.0.2", 5432) b.AddEndpoint("172.17.0.3", 5432) b.CheckInterval = "2s" @@ -1164,7 +1164,7 @@ listen _tcp_pq_5432 // 2 { doconfig: func(c *testConfig) { - b := c.config.AcquireTCPBackend("pq", 5432) + b := c.config.TCPBackends().Acquire("pq", 5432) b.AddEndpoint("172.17.0.2", 5432) b.SSL.Filename = "/var/haproxy/ssl/pq.pem" b.ProxyProt.EncodeVersion = "v2" @@ -1178,7 +1178,7 @@ listen _tcp_pq_5432 // 3 { doconfig: func(c *testConfig) { - b := c.config.AcquireTCPBackend("pq", 5432) + b := c.config.TCPBackends().Acquire("pq", 5432) b.AddEndpoint("172.17.0.2", 5432) b.SSL.Filename = "/var/haproxy/ssl/pq.pem" b.ProxyProt.Decode = true @@ -1195,7 +1195,7 @@ listen _tcp_pq_5432 // 4 { doconfig: func(c *testConfig) { - b := c.config.AcquireTCPBackend("pq", 5432) + b := c.config.TCPBackends().Acquire("pq", 5432) b.AddEndpoint("172.17.0.2", 5432) b.SSL.Filename = "/var/haproxy/ssl/pq.pem" b.SSL.CAFilename = "/var/haproxy/ssl/pqca.pem" @@ -1210,7 +1210,7 @@ listen _tcp_pq_5432 // 5 { doconfig: func(c *testConfig) { - b := c.config.AcquireTCPBackend("pq", 5432) + b := c.config.TCPBackends().Acquire("pq", 5432) b.AddEndpoint("172.17.0.2", 5432) b.SSL.Filename = "/var/haproxy/ssl/pq.pem" b.SSL.CAFilename = "/var/haproxy/ssl/pqca.pem" @@ -2307,7 +2307,7 @@ userlist default_auth2 h.AddPath(b, "/admin") for _, list := range test.lists { - c.config.AddUserlist(list.name, list.users) + c.config.Userlists().Replace(list.name, list.users) } b.AuthHTTP = []*hatypes.BackendConfigAuth{ { @@ -2392,7 +2392,7 @@ frontend _front_http h = c.config.Hosts().AcquireHost("d1.local") h.AddPath(b, "/") - acme := c.config.Acme() + acme := &c.config.Global().Acme acme.Enabled = true acme.Prefix = "/.acme" acme.Socket = "/run/acme.sock" @@ -2861,8 +2861,8 @@ func setup(t *testing.T) *testConfig { mapsTemplate: instance.mapsTemplate, mapsDir: tempdir, }) - instance.curConfig = config - config.ConfigDefaultX509Cert("/var/haproxy/ssl/certs/default.pem") + instance.config = config + config.frontend.DefaultCert = "/var/haproxy/ssl/certs/default.pem" c := &testConfig{ t: t, logger: logger, @@ -2882,16 +2882,6 @@ func (c *testConfig) teardown() { } } -func (c *testConfig) newConfig() Config { - config := createConfig(options{ - mapsTemplate: c.instance.(*instance).mapsTemplate, - mapsDir: c.tempdir, - }) - config.ConfigDefaultX509Cert("/var/haproxy/ssl/certs/default.pem") - c.configGlobal(config.Global()) - return config -} - func (c *testConfig) configGlobal(global *hatypes.Global) { global.AdminSocket = "/var/run/haproxy.sock" global.Bind.HTTPBind = ":80" diff --git a/pkg/haproxy/types/backend.go b/pkg/haproxy/types/backend.go index f1ede83d3..c581671c0 100644 --- a/pkg/haproxy/types/backend.go +++ b/pkg/haproxy/types/backend.go @@ -29,6 +29,17 @@ func NewBackendPaths(paths ...*BackendPath) BackendPaths { return b } +// BackendID ... +func (b *Backend) BackendID() BackendID { + // IMPLEMENT as pointer + // TODO immutable internal state + return BackendID{ + Namespace: b.Namespace, + Name: b.Name, + Port: b.Port, + } +} + // FindEndpoint ... func (b *Backend) FindEndpoint(target string) *Endpoint { for _, endpoint := range b.Endpoints { diff --git a/pkg/haproxy/types/backends.go b/pkg/haproxy/types/backends.go index 2a1e0d70a..555afdbfb 100644 --- a/pkg/haproxy/types/backends.go +++ b/pkg/haproxy/types/backends.go @@ -23,13 +23,56 @@ import ( // CreateBackends ... func CreateBackends() *Backends { return &Backends{ - itemsmap: map[string]*Backend{}, + items: map[string]*Backend{}, + itemsAdd: map[string]*Backend{}, + itemsDel: map[string]*Backend{}, } } // Items ... -func (b *Backends) Items() []*Backend { - return b.itemslist +func (b *Backends) Items() map[string]*Backend { + return b.items +} + +// ItemsAdd ... +func (b *Backends) ItemsAdd() map[string]*Backend { + return b.itemsAdd +} + +// ItemsDel ... +func (b *Backends) ItemsDel() map[string]*Backend { + return b.itemsDel +} + +// Commit ... +func (b *Backends) Commit() { + b.itemsAdd = map[string]*Backend{} + b.itemsDel = map[string]*Backend{} +} + +// Changed ... +func (b *Backends) Changed() bool { + return len(b.itemsAdd) > 0 || len(b.itemsDel) > 0 +} + +// BuildSortedItems ... +func (b *Backends) BuildSortedItems() []*Backend { + items := make([]*Backend, len(b.items)) + var i int + for _, item := range b.items { + items[i] = item + i++ + } + sort.Slice(items, func(i, j int) bool { + if items[i] == b.defaultBackend { + return false + } + if items[j] == b.defaultBackend { + return true + } + return items[i].ID < items[j].ID + }) + return items } // AcquireBackend ... @@ -38,20 +81,30 @@ func (b *Backends) AcquireBackend(namespace, name, port string) *Backend { return backend } backend := createBackend(namespace, name, port) - // Store backends on slice and map data structure. - // The slice has the order and the map has the index. - // TODO current approach is using the double of the memory - // on behalf of speed. Map only is doable? Another approach? - // See also hosts.AcquireHost(). - b.itemsmap[backend.ID] = backend - b.itemslist = append(b.itemslist, backend) - b.sortBackends() + b.items[backend.ID] = backend + b.itemsAdd[backend.ID] = backend return backend } // FindBackend ... func (b *Backends) FindBackend(namespace, name, port string) *Backend { - return b.itemsmap[buildID(namespace, name, port)] + return b.items[buildID(namespace, name, port)] +} + +// FindBackendID ... +func (b *Backends) FindBackendID(backendID BackendID) *Backend { + return b.items[backendID.String()] +} + +// RemoveAll ... +func (b *Backends) RemoveAll(backendID []BackendID) { + for _, backend := range backendID { + id := backend.String() + if item, found := b.items[id]; found { + b.itemsDel[id] = item + delete(b.items, id) + } + } } // DefaultBackend ... @@ -69,19 +122,13 @@ func (b *Backends) SetDefaultBackend(defaultBackend *Backend) { if b.defaultBackend != nil { b.defaultBackend.ID = "_default_backend" } - b.sortBackends() } -func (b *Backends) sortBackends() { - sort.Slice(b.itemslist, func(i, j int) bool { - if b.itemslist[i] == b.defaultBackend { - return false - } - if b.itemslist[j] == b.defaultBackend { - return true - } - return b.itemslist[i].ID < b.itemslist[j].ID - }) +func (b BackendID) String() string { + if b.id == "" { + b.id = b.Namespace + "_" + b.Name + "_" + b.Port + } + return b.id } func createBackend(namespace, name, port string) *Backend { diff --git a/pkg/haproxy/types/global.go b/pkg/haproxy/types/global.go index 04f29de00..f571706b0 100644 --- a/pkg/haproxy/types/global.go +++ b/pkg/haproxy/types/global.go @@ -18,20 +18,106 @@ package types import ( "fmt" + "reflect" + "sort" + "strings" ) -// AddDomains ... -func (acme *AcmeData) AddDomains(storage string, domains []string) { - if acme.Certs == nil { - acme.Certs = map[string]map[string]struct{}{} +// Storages ... +func (acme *AcmeData) Storages() *AcmeStorages { + if acme.storages == nil { + acme.storages = &AcmeStorages{ + items: map[string]*AcmeCerts{}, + itemsAdd: map[string]*AcmeCerts{}, + itemsDel: map[string]*AcmeCerts{}, + } } - certs, found := acme.Certs[storage] + return acme.storages +} + +// Acquire ... +func (c *AcmeStorages) Acquire(name string) *AcmeCerts { + storage, found := c.items[name] if !found { - certs = map[string]struct{}{} - acme.Certs[storage] = certs + storage = &AcmeCerts{ + certs: map[string]struct{}{}, + } + c.items[name] = storage + c.itemsAdd[name] = storage + } + return storage +} + +// Updated ... +func (c *AcmeStorages) Updated() bool { + c.shrink() + return len(c.itemsAdd) > 0 || len(c.itemsDel) > 0 +} + +// BuildAcmeStorages ... +func (c *AcmeStorages) BuildAcmeStorages() []string { + return buildAcmeStorages(c.items) +} + +// BuildAcmeStoragesAdd ... +func (c *AcmeStorages) BuildAcmeStoragesAdd() []string { + c.shrink() + return buildAcmeStorages(c.itemsAdd) +} + +// BuildAcmeStoragesDel ... +func (c *AcmeStorages) BuildAcmeStoragesDel() []string { + c.shrink() + return buildAcmeStorages(c.itemsDel) +} + +func buildAcmeStorages(items map[string]*AcmeCerts) []string { + storages := make([]string, len(items)) + i := 0 + for name := range items { + item := items[name] + certs := make([]string, len(item.certs)) + j := 0 + for cert := range item.certs { + certs[j] = cert + j++ + } + sort.Strings(certs) + storages[i] = name + "," + strings.Join(certs, ",") + i++ } + return storages +} + +func (c *AcmeStorages) shrink() { + for item, del := range c.itemsDel { + if add, found := c.itemsAdd[item]; found && reflect.DeepEqual(add, del) { + delete(c.itemsAdd, item) + delete(c.itemsDel, item) + } + } +} + +// RemoveAll ... +func (c *AcmeStorages) RemoveAll(names []string) { + for _, name := range names { + if item, found := c.items[name]; found { + c.itemsDel[name] = item + delete(c.items, name) + } + } +} + +// Commit ... +func (c *AcmeStorages) Commit() { + c.itemsAdd = map[string]*AcmeCerts{} + c.itemsDel = map[string]*AcmeCerts{} +} + +// AddDomains ... +func (c *AcmeCerts) AddDomains(domains []string) { for _, domain := range domains { - certs[domain] = struct{}{} + c.certs[domain] = struct{}{} } } @@ -47,10 +133,6 @@ func (dns *DNSNameserver) String() string { return fmt.Sprintf("%+v", *dns) } -func (u *Userlist) String() string { - return fmt.Sprintf("%+v", *u) -} - // ShareHTTPPort ... func (b GlobalBindConfig) ShareHTTPPort() bool { return b.HasFrontingProxy() && b.HTTPBind == b.FrontingBind diff --git a/pkg/haproxy/types/global_test.go b/pkg/haproxy/types/global_test.go index a2030d74a..82ca65067 100644 --- a/pkg/haproxy/types/global_test.go +++ b/pkg/haproxy/types/global_test.go @@ -18,21 +18,22 @@ package types import ( "reflect" + "sort" "testing" ) -func TestAcmeAddDomain(t *testing.T) { +func TestBuildAcmeStorages(t *testing.T) { testCases := []struct { certs [][]string - expected map[string]map[string]struct{} + expected []string }{ // 0 { certs: [][]string{ {"cert1", "d1.local"}, }, - expected: map[string]map[string]struct{}{ - "cert1": {"d1.local": {}}, + expected: []string{ + "cert1,d1.local", }, }, // 1 @@ -41,8 +42,8 @@ func TestAcmeAddDomain(t *testing.T) { {"cert1", "d1.local", "d2.local"}, {"cert1", "d2.local", "d3.local"}, }, - expected: map[string]map[string]struct{}{ - "cert1": {"d1.local": {}, "d2.local": {}, "d3.local": {}}, + expected: []string{ + "cert1,d1.local,d2.local,d3.local", }, }, // 2 @@ -51,19 +52,97 @@ func TestAcmeAddDomain(t *testing.T) { {"cert1", "d1.local", "d2.local"}, {"cert2", "d2.local", "d3.local"}, }, - expected: map[string]map[string]struct{}{ - "cert1": {"d1.local": {}, "d2.local": {}}, - "cert2": {"d2.local": {}, "d3.local": {}}, + expected: []string{ + "cert1,d1.local,d2.local", + "cert2,d2.local,d3.local", }, }, } for i, test := range testCases { acme := AcmeData{} for _, cert := range test.certs { - acme.AddDomains(cert[0], cert[1:]) + acme.Storages().Acquire(cert[0]).AddDomains(cert[1:]) } - if !reflect.DeepEqual(acme.Certs, test.expected) { - t.Errorf("acme certs differs on %d - expected: %+v, actual: %+v", i, test.expected, acme.Certs) + storages := acme.Storages().BuildAcmeStorages() + sort.Strings(storages) + if !reflect.DeepEqual(storages, test.expected) { + t.Errorf("acme certs differs on %d - expected: %+v, actual: %+v", i, test.expected, storages) + } + } +} + +func TestShrink(t *testing.T) { + d1 := map[string]struct{}{"d1.local": {}} + d2 := map[string]struct{}{"d2.local": {}} + testCases := []struct { + itemAdd, itemDel map[string]*AcmeCerts + expAdd, expDel map[string]*AcmeCerts + }{ + // 0 + { + expAdd: map[string]*AcmeCerts{}, + expDel: map[string]*AcmeCerts{}, + }, + // 1 + { + itemAdd: map[string]*AcmeCerts{"cert1": {d1}}, + expAdd: map[string]*AcmeCerts{"cert1": {d1}}, + expDel: map[string]*AcmeCerts{}, + }, + // 2 + { + itemAdd: map[string]*AcmeCerts{"cert1": {d1}}, + itemDel: map[string]*AcmeCerts{"cert1": {d1}}, + expAdd: map[string]*AcmeCerts{}, + expDel: map[string]*AcmeCerts{}, + }, + // 3 + { + itemAdd: map[string]*AcmeCerts{ + "cert1": {d1}, + "cert2": {d1}, + }, + itemDel: map[string]*AcmeCerts{ + "cert1": {d1}, + "cert2": {d2}, + }, + expAdd: map[string]*AcmeCerts{ + "cert2": {d1}, + }, + expDel: map[string]*AcmeCerts{ + "cert2": {d2}, + }, + }, + // 4 + { + itemAdd: map[string]*AcmeCerts{ + "cert1": {d1}, + "cert2": {d1}, + }, + itemDel: map[string]*AcmeCerts{ + "cert1": {d1}, + }, + expAdd: map[string]*AcmeCerts{ + "cert2": {d1}, + }, + expDel: map[string]*AcmeCerts{}, + }, + } + for i, test := range testCases { + acme := AcmeData{} + storages := acme.Storages() + if test.itemAdd != nil { + storages.itemsAdd = test.itemAdd + } + if test.itemDel != nil { + storages.itemsDel = test.itemDel + } + storages.shrink() + if !reflect.DeepEqual(storages.itemsAdd, test.expAdd) { + t.Errorf("itemAdd differs on %d - expected: %+v, actual: %+v", i, test.expAdd, storages.itemsAdd) + } + if !reflect.DeepEqual(storages.itemsDel, test.expDel) { + t.Errorf("itemDel differs on %d - expected: %+v, actual: %+v", i, test.expDel, storages.itemsDel) } } } diff --git a/pkg/haproxy/types/host.go b/pkg/haproxy/types/host.go index a892a081f..494eac5df 100644 --- a/pkg/haproxy/types/host.go +++ b/pkg/haproxy/types/host.go @@ -18,13 +18,16 @@ package types import ( "fmt" + "reflect" "sort" ) // CreateHosts ... func CreateHosts() *Hosts { return &Hosts{ - itemsmap: map[string]*Host{}, + items: map[string]*Host{}, + itemsAdd: map[string]*Host{}, + itemsDel: map[string]*Host{}, } } @@ -35,15 +38,8 @@ func (h *Hosts) AcquireHost(hostname string) *Host { } host := h.createHost(hostname) if host.Hostname != "*" { - // Here we store a just created Host. Both slice and map. - // The slice has the order and the map has the index. - // TODO current approach is using the double of the memory - // on behalf of speed. Map only is doable? Another approach? - h.itemsmap[hostname] = host - h.itemslist = append(h.itemslist, host) - sort.Slice(h.itemslist, func(i, j int) bool { - return h.itemslist[i].Hostname < h.itemslist[j].Hostname - }) + h.items[hostname] = host + h.itemsAdd[hostname] = host } else { h.defaultHost = host } @@ -55,7 +51,28 @@ func (h *Hosts) FindHost(hostname string) *Host { if hostname == "*" && h.defaultHost != nil { return h.defaultHost } - return h.itemsmap[hostname] + return h.items[hostname] +} + +// RemoveAll ... +func (h *Hosts) RemoveAll(hostnames []string) { + for _, hostname := range hostnames { + if item, found := h.items[hostname]; found { + h.itemsDel[hostname] = item + delete(h.items, hostname) + } + } +} + +// Commit ... +func (h *Hosts) Commit() { + h.itemsAdd = map[string]*Host{} + h.itemsDel = map[string]*Host{} +} + +// Changed ... +func (h *Hosts) Changed() bool { + return !reflect.DeepEqual(h.itemsAdd, h.itemsDel) } func (h *Hosts) createHost(hostname string) *Host { @@ -65,9 +82,36 @@ func (h *Hosts) createHost(hostname string) *Host { } } +// BuildSortedItems ... +func (h *Hosts) BuildSortedItems() []*Host { + items := make([]*Host, len(h.items)) + var i int + for _, item := range h.items { + items[i] = item + i++ + } + sort.Slice(items, func(i, j int) bool { + return items[i].Hostname < items[j].Hostname + }) + if len(items) == 0 { + return nil + } + return items +} + // Items ... -func (h *Hosts) Items() []*Host { - return h.itemslist +func (h *Hosts) Items() map[string]*Host { + return h.items +} + +// ItemsAdd ... +func (h *Hosts) ItemsAdd() map[string]*Host { + return h.itemsAdd +} + +// ItemsDel ... +func (h *Hosts) ItemsDel() map[string]*Host { + return h.itemsDel } // DefaultHost ... @@ -82,12 +126,12 @@ func (h *Hosts) HasSSLPassthrough() bool { // HasHTTP ... func (h *Hosts) HasHTTP() bool { - return len(h.itemslist) > h.sslPassthroughCount + return len(h.items) > h.sslPassthroughCount } // HasTLSAuth ... func (h *Hosts) HasTLSAuth() bool { - for _, host := range h.itemslist { + for _, host := range h.items { if host.HasTLSAuth() { return true } @@ -97,7 +141,7 @@ func (h *Hosts) HasTLSAuth() bool { // HasTLSMandatory ... func (h *Hosts) HasTLSMandatory() bool { - for _, host := range h.itemslist { + for _, host := range h.items { if host.HasTLSAuth() && !host.TLS.CAVerifyOptional { return true } @@ -107,7 +151,7 @@ func (h *Hosts) HasTLSMandatory() bool { // HasVarNamespace ... func (h *Hosts) HasVarNamespace() bool { - for _, host := range h.itemslist { + for _, host := range h.items { if host.VarNamespace { return true } diff --git a/pkg/haproxy/types/tcpbackend.go b/pkg/haproxy/types/tcpbackend.go index d9aa75693..5ee7eb2ae 100644 --- a/pkg/haproxy/types/tcpbackend.go +++ b/pkg/haproxy/types/tcpbackend.go @@ -18,8 +18,75 @@ package types import ( "fmt" + "reflect" + "sort" ) +// CreateTCPBackends ... +func CreateTCPBackends() *TCPBackends { + return &TCPBackends{ + items: map[int]*TCPBackend{}, + itemsAdd: map[int]*TCPBackend{}, + itemsDel: map[int]*TCPBackend{}, + } +} + +// Acquire ... +func (b *TCPBackends) Acquire(servicename string, port int) *TCPBackend { + if backend, found := b.items[port]; found { + backend.Name = servicename + return backend + } + backend := &TCPBackend{ + Name: servicename, + Port: port, + } + b.items[port] = backend + b.itemsAdd[port] = backend + return backend +} + +// BuildSortedItems ... +func (b *TCPBackends) BuildSortedItems() []*TCPBackend { + items := make([]*TCPBackend, len(b.items)) + var i int + for _, item := range b.items { + items[i] = item + i++ + } + sort.Slice(items, func(i, j int) bool { + back1 := items[i] + back2 := items[j] + if back1.Name == back2.Name { + return back1.Port < back2.Port + } + return back1.Name < back2.Name + }) + if len(items) == 0 { + return nil + } + return items +} + +// Changed ... +func (b *TCPBackends) Changed() bool { + return !reflect.DeepEqual(b.itemsAdd, b.itemsDel) +} + +// Commit ... +func (b *TCPBackends) Commit() { + b.itemsAdd = map[int]*TCPBackend{} + b.itemsDel = map[int]*TCPBackend{} +} + +// RemoveAll ... +func (b *TCPBackends) RemoveAll() { + for port, item := range b.items { + b.itemsDel[port] = item + delete(b.items, port) + } +} + // AddEndpoint ... func (b *TCPBackend) AddEndpoint(ip string, port int) *TCPEndpoint { ep := &TCPEndpoint{ diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 7aa0c1352..3d46ae057 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -22,13 +22,23 @@ import ( // AcmeData ... type AcmeData struct { - Certs map[string]map[string]struct{} + storages *AcmeStorages Emails string Endpoint string Expiring time.Duration TermsAgreed bool } +// AcmeStorages ... +type AcmeStorages struct { + items, itemsAdd, itemsDel map[string]*AcmeCerts +} + +// AcmeCerts ... +type AcmeCerts struct { + certs map[string]struct{} +} + // Acme ... type Acme struct { Enabled bool @@ -49,6 +59,7 @@ type Global struct { ModSecurity ModSecurityConfig Cookie CookieConfig DrainSupport DrainConfig + Acme Acme ForwardFor string LoadServerState bool AdminSocket string @@ -199,6 +210,11 @@ type ModSecurityTimeoutConfig struct { Processing string } +// TCPBackends ... +type TCPBackends struct { + items, itemsAdd, itemsDel map[int]*TCPBackend +} + // TCPBackend ... type TCPBackend struct { Name string @@ -277,12 +293,13 @@ type Frontend struct { BindSocket string BindID int AcceptProxy bool + DefaultCert string } // Hosts ... type Hosts struct { - itemslist []*Host - itemsmap map[string]*Host + items, itemsAdd, itemsDel map[string]*Host + // defaultHost *Host // sslPassthroughCount int @@ -361,16 +378,26 @@ const ( // Backends ... type Backends struct { - itemslist []*Backend - itemsmap map[string]*Backend + items, itemsAdd, itemsDel map[string]*Backend + // defaultBackend *Backend } +// BackendID ... +type BackendID struct { + id string + Namespace string + Name string + Port string +} + // Backend ... type Backend struct { // // core config // + // IMPLEMENT + // use BackendID ID string Namespace string Name string @@ -636,6 +663,11 @@ type WAF struct { Module string } +// Userlists ... +type Userlists struct { + items, itemsAdd, itemsDel map[string]*Userlist +} + // Userlist ... type Userlist struct { Name string diff --git a/pkg/haproxy/types/userlist.go b/pkg/haproxy/types/userlist.go new file mode 100644 index 000000000..e2c9efc1f --- /dev/null +++ b/pkg/haproxy/types/userlist.go @@ -0,0 +1,94 @@ +/* +Copyright 2020 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "fmt" + "reflect" + "sort" +) + +// CreateUserlists ... +func CreateUserlists() *Userlists { + return &Userlists{ + items: map[string]*Userlist{}, + itemsAdd: map[string]*Userlist{}, + itemsDel: map[string]*Userlist{}, + } +} + +// Replace ... +func (u *Userlists) Replace(name string, users []User) *Userlist { + userlist := &Userlist{ + Name: name, + Users: users, + } + sort.Slice(users, func(i, j int) bool { + return users[i].Name < users[j].Name + }) + u.items[name] = userlist + u.itemsAdd[name] = userlist + return userlist +} + +// Find ... +func (u *Userlists) Find(name string) *Userlist { + return u.items[name] +} + +// BuildSortedItems ... +func (u *Userlists) BuildSortedItems() []*Userlist { + items := make([]*Userlist, len(u.items)) + var i int + for _, item := range u.items { + items[i] = item + i++ + } + sort.Slice(items, func(i, j int) bool { + return items[i].Name < items[j].Name + }) + if len(items) == 0 { + return nil + } + return items +} + +// RemoveAll ... +func (u *Userlists) RemoveAll(userlists []string) { + for _, userlist := range userlists { + if item, found := u.items[userlist]; found { + u.itemsDel[userlist] = item + delete(u.items, userlist) + } + } + +} + +// Changed ... +func (u *Userlists) Changed() bool { + return !reflect.DeepEqual(u.itemsAdd, u.itemsDel) +} + +// Commit ... +func (u *Userlists) Commit() { + u.itemsAdd = map[string]*Userlist{} + u.itemsDel = map[string]*Userlist{} +} + +func (u *Userlist) String() string { + return fmt.Sprintf("%+v", *u) +} diff --git a/rootfs/etc/haproxy/template/haproxy.tmpl b/rootfs/etc/haproxy/template/haproxy.tmpl index f7f945fc8..c644dc877 100644 --- a/rootfs/etc/haproxy/template/haproxy.tmpl +++ b/rootfs/etc/haproxy/template/haproxy.tmpl @@ -143,7 +143,7 @@ resolvers {{ $resolver.Name }} {{- end }} {{- end }} -{{- $userlists := $cfg.Userlists }} +{{- $userlists := $cfg.Userlists.BuildSortedItems }} {{- if $userlists }} # # # # # # # # # # # # # # # # # # # @@ -158,7 +158,8 @@ userlist {{ $userlist.Name }} {{- end }} {{- end }} -{{- if $cfg.TCPBackends }} +{{- $tcpbackends := $cfg.TCPBackends.BuildSortedItems}} +{{- if $tcpbackends }} # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -168,7 +169,7 @@ userlist {{ $userlist.Name }} # # # -{{- range $backend := $cfg.TCPBackends }} +{{- range $backend := $tcpbackends }} listen _tcp_{{ $backend.Name }}_{{ $backend.Port }} {{- $ssl := $backend.SSL }} bind {{ $global.Bind.TCPBindIP }}:{{ $backend.Port }} @@ -204,7 +205,8 @@ listen _tcp_{{ $backend.Name }}_{{ $backend.Port }} {{- end }}{{/* range TCPBackends */}} {{- end }}{{/* if has TCPBackend */}} -{{- if $backends.Items }} +{{- $backendItems := $backends.BuildSortedItems }} +{{- if $backendItems }} # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -213,7 +215,7 @@ listen _tcp_{{ $backend.Name }}_{{ $backend.Port }} # # BACKENDS # # # -{{- range $backend := $backends.Items }} +{{- range $backend := $backendItems }} backend {{ $backend.ID }} mode {{ if $backend.ModeTCP }}tcp{{ else }}http{{ end }} {{- if $backend.BalanceAlgorithm }} @@ -553,7 +555,7 @@ backend {{ $backend.ID }} {{- end }} {{- end }} -{{- end }}{{/* if has backends.Items */}} +{{- end }}{{/* if backendItems */}} {{- define "backend" }} {{- $backend := .p1 }} @@ -590,7 +592,7 @@ backend {{ $backend.ID }} {{- end }} {{- end }} -{{- if $cfg.Acme.Enabled }} +{{- if $global.Acme.Enabled }} # # # # # # # # # # # # # # # # # # # # # @@ -598,7 +600,7 @@ backend {{ $backend.ID }} # backend _acme_challenge mode http - server _acme_server unix@{{ $cfg.Acme.Socket }} + server _acme_server unix@{{ $global.Acme.Socket }} {{- end }} {{- if not $backends.DefaultBackend }} @@ -731,15 +733,15 @@ frontend _front_http {{- end }} {{- /*------------------------------------*/}} -{{- if $cfg.Acme.Enabled }} - acl acme-challenge path_beg {{ $cfg.Acme.Prefix }} +{{- if $global.Acme.Enabled }} + acl acme-challenge path_beg {{ $global.Acme.Prefix }} {{- end }} {{- /*------------------------------------*/}} http-request set-var(req.base) base,lower,regsub(:[0-9]+/,/) {{- /*------------------------------------*/}} -{{- $acmeexclusive := and $cfg.Acme.Enabled (not $cfg.Acme.Shared) }} +{{- $acmeexclusive := and $global.Acme.Enabled (not $global.Acme.Shared) }} {{- if not $frontingIgnoreProto }} {{- if $fmaps.HTTPSRedirMap.HasRegex }} http-request set-var(req.redir) @@ -822,7 +824,7 @@ frontend _front_http use_backend _acme_challenge if acme-challenge {{- end }} use_backend %[var(req.backend)] if { var(req.backend) -m found } -{{- if and $cfg.Acme.Enabled $cfg.Acme.Shared }} +{{- if and $global.Acme.Enabled $global.Acme.Shared }} use_backend _acme_challenge if acme-challenge {{- end }}