From 2a24f12e0cf478d3e501fbf09a6805f16892fe5b Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Mon, 27 Apr 2020 22:16:19 -0300 Subject: [PATCH] implement model update Improves the time of object parsing on big k8s clusters - about 1000+ ingress objects which references 1000+ services - using a updatable haproxy model and a object<->model tracking. The implementation follows this approach: * track changes on each relevant object type - currently ingress, service, endpoint, secret, configmap, and also pod if drain-support is used; * every object tracks a host and/or backend in the haproxy model; * if the object is removed or updated, only the tracked host and/or backend is rebuilt; * dynupdate doesn't compare old/cur state anymore, but instead dirty and new host/backend state. This update has the majority of the job: notification improvement, object<->model tracking and converter changes. Missing dirty/new state in the haproxy model, concurrency improvements, dynupdate refactor and some minor adjusts. --- pkg/common/ingress/controller/backend_ssl.go | 3 - pkg/common/ingress/controller/controller.go | 1 - pkg/controller/cache.go | 297 ++++++++++- pkg/controller/controller.go | 100 +--- pkg/controller/listers.go | 82 +-- pkg/converters/helper_test/cachemock.go | 52 ++ pkg/converters/helper_test/trackermock.go | 55 +++ pkg/converters/ingress/annotations/mapper.go | 7 +- pkg/converters/ingress/ingress.go | 211 +++++++- pkg/converters/ingress/ingress_test.go | 8 +- pkg/converters/ingress/tracker/tracker.go | 379 ++++++++++++++ .../ingress/tracker/tracker_test.go | 466 ++++++++++++++++++ pkg/converters/ingress/types/options.go | 1 + pkg/converters/types/interfaces.go | 38 ++ pkg/haproxy/config.go | 5 + pkg/haproxy/types/backend.go | 15 + pkg/haproxy/types/backends.go | 13 + pkg/haproxy/types/host.go | 5 + pkg/haproxy/types/types.go | 9 + 19 files changed, 1601 insertions(+), 146 deletions(-) create mode 100644 pkg/converters/helper_test/trackermock.go create mode 100644 pkg/converters/ingress/tracker/tracker.go create mode 100644 pkg/converters/ingress/tracker/tracker_test.go diff --git a/pkg/common/ingress/controller/backend_ssl.go b/pkg/common/ingress/controller/backend_ssl.go index 2af452b7e..9a420540e 100644 --- a/pkg/common/ingress/controller/backend_ssl.go +++ b/pkg/common/ingress/controller/backend_ssl.go @@ -43,7 +43,6 @@ func (ic *GenericController) SyncSecret(key string) { cert, err := ic.getPemCertificate(secret) if err != nil { glog.V(3).Infof("syncing a non ca/crt secret %v", key) - ic.newctrl.Notify() return } @@ -59,7 +58,6 @@ func (ic *GenericController) SyncSecret(key string) { ic.sslCertTracker.Update(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) - ic.newctrl.Notify() return } @@ -67,7 +65,6 @@ func (ic *GenericController) SyncSecret(key string) { ic.sslCertTracker.Add(key, cert) // this update must trigger an update // (like an update event from a change in Ingress) - ic.newctrl.Notify() } // getPemCertificate receives a secret, and creates a ingress.SSLCert as return. diff --git a/pkg/common/ingress/controller/controller.go b/pkg/common/ingress/controller/controller.go index b04f66a61..3ee279487 100644 --- a/pkg/common/ingress/controller/controller.go +++ b/pkg/common/ingress/controller/controller.go @@ -40,7 +40,6 @@ import ( type NewCtrlIntf interface { GetIngressList() ([]*extensions.Ingress, error) GetSecret(name string) (*apiv1.Secret, error) - Notify() } // GenericController holds the boilerplate code required to build an Ingress controlller. diff --git a/pkg/controller/cache.go b/pkg/controller/cache.go index 6bb1b7d5d..c1258c6ed 100644 --- a/pkg/controller/cache.go +++ b/pkg/controller/cache.go @@ -25,18 +25,26 @@ import ( "fmt" "os" "strings" + "sync" + "time" api "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" "github.com/jcmoraisjr/haproxy-ingress/pkg/acme" cfile "github.com/jcmoraisjr/haproxy-ingress/pkg/common/file" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress/controller" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/net/ssl" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + "github.com/jcmoraisjr/haproxy-ingress/pkg/types" + "github.com/jcmoraisjr/haproxy-ingress/pkg/utils" ) const dhparamFilename = "dhparam.pem" @@ -46,11 +54,43 @@ type k8scache struct { listers *listers controller *controller.GenericController crossNS bool + globalConfigMapKey string + tcpConfigMapKey string acmeSecretKeyName string acmeTokenConfigmapName string + // + updateQueue utils.Queue + stateMutex sync.RWMutex + clear bool + needResync bool + // + globalConfigMapData map[string]string + tcpConfigMapData map[string]string + newGlobalConfigMapData map[string]string + newTCPConfigMapData map[string]string + // + delIngresses []*extensions.Ingress + updIngresses []*extensions.Ingress + addIngresses []*extensions.Ingress + newEndpoints []*api.Endpoints + delServices []*api.Service + updServices []*api.Service + addServices []*api.Service + delSecrets []*api.Secret + updSecrets []*api.Secret + addSecrets []*api.Secret + newPods []*api.Pod + // } -func newCache(client k8s.Interface, listers *listers, controller *controller.GenericController) *k8scache { +func createCache( + logger types.Logger, + client k8s.Interface, + controller *controller.GenericController, + updateQueue utils.Queue, + watchNamespace string, + resync time.Duration, +) *k8scache { namespace := os.Getenv("POD_NAMESPACE") if namespace == "" { // TODO implement a smart fallback or error checking @@ -70,14 +110,36 @@ func newCache(client k8s.Interface, listers *listers, controller *controller.Gen if !strings.Contains(acmeTokenConfigmapName, "/") { acmeTokenConfigmapName = namespace + "/" + acmeTokenConfigmapName } - return &k8scache{ + globalConfigMapName := cfg.ConfigMapName + tcpConfigMapName := cfg.TCPConfigMapName + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(logger.Info) + eventBroadcaster.StartRecordingToSink(&typedv1.EventSinkImpl{ + Interface: client.CoreV1().Events(watchNamespace), + }) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, api.EventSource{ + Component: "ingress-controller", + }) + cache := &k8scache{ client: client, - listers: listers, controller: controller, crossNS: cfg.AllowCrossNamespace, + globalConfigMapKey: globalConfigMapName, + tcpConfigMapKey: tcpConfigMapName, acmeSecretKeyName: acmeSecretKeyName, acmeTokenConfigmapName: acmeTokenConfigmapName, - } + stateMutex: sync.RWMutex{}, + updateQueue: updateQueue, + clear: true, + needResync: false, + } + // TODO I'm a circular reference, can you fix me? + cache.listers = createListers(cache, logger, recorder, client, watchNamespace, resync) + return cache +} + +func (c *k8scache) RunAsync(stopCh <-chan struct{}) { + c.listers.RunAsync(stopCh) } func (c *k8scache) GetIngressPodName() (namespace, podname string, err error) { @@ -92,6 +154,18 @@ func (c *k8scache) GetIngressPodName() (namespace, podname string, err error) { return namespace, podname, nil } +func (c *k8scache) GetIngress(ingressName string) (*extensions.Ingress, error) { + namespace, name, err := cache.SplitMetaNamespaceKey(ingressName) + if err != nil { + return nil, err + } + return c.listers.ingressLister.Ingresses(namespace).Get(name) +} + +func (c *k8scache) GetIngressList() ([]*extensions.Ingress, error) { + return c.listers.ingressLister.List(labels.Everything()) +} + func (c *k8scache) GetService(serviceName string) (*api.Service, error) { namespace, name, err := cache.SplitMetaNamespaceKey(serviceName) if err != nil { @@ -423,3 +497,218 @@ func (c *k8scache) CreateOrUpdateConfigMap(cm *api.ConfigMap) (err error) { } return err } + +// implements ListerEvents +func (c *k8scache) IsValidIngress(ing *extensions.Ingress) bool { + return c.controller.IsValidClass(ing) +} + +// implements ListerEvents +func (c *k8scache) IsValidConfigMap(cm *api.ConfigMap) bool { + key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) + return key == c.globalConfigMapKey || key == c.tcpConfigMapKey +} + +// implements ListerEvents +func (c *k8scache) Notify(old, cur interface{}) { + // IMPLEMENT + // maintain a list of changed objects only if partial parsing + // is being used -- SyncNewObjects() is being called + c.stateMutex.Lock() + defer c.stateMutex.Unlock() + if old == nil && cur == nil { + c.needResync = true + } + if old != nil { + switch old.(type) { + case *extensions.Ingress: + if cur == nil { + c.delIngresses = append(c.delIngresses, old.(*extensions.Ingress)) + } + case *api.Service: + if cur == nil { + c.delServices = append(c.delServices, old.(*api.Service)) + } + case *api.Secret: + if cur == nil { + secret := old.(*api.Secret) + c.delSecrets = append(c.delSecrets, secret) + c.controller.DeleteSecret(fmt.Sprintf("%s/%s", secret.Namespace, secret.Name)) + } + } + } + if cur != nil { + switch cur.(type) { + case *extensions.Ingress: + ing := cur.(*extensions.Ingress) + if old == nil { + c.addIngresses = append(c.addIngresses, ing) + } else { + c.updIngresses = append(c.updIngresses, ing) + } + case *api.Endpoints: + c.newEndpoints = append(c.newEndpoints, cur.(*api.Endpoints)) + case *api.Service: + svc := cur.(*api.Service) + if old == nil { + c.addServices = append(c.addServices, svc) + } else { + c.updServices = append(c.updServices, svc) + } + case *api.Secret: + secret := cur.(*api.Secret) + if old == nil { + c.addSecrets = append(c.addSecrets, secret) + } else { + c.updSecrets = append(c.updSecrets, secret) + } + case *api.ConfigMap: + cm := cur.(*api.ConfigMap) + key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) + switch key { + case c.globalConfigMapKey: + c.newGlobalConfigMapData = cm.Data + case c.tcpConfigMapKey: + c.newTCPConfigMapData = cm.Data + } + case *api.Pod: + c.newPods = append(c.newPods, cur.(*api.Pod)) + } + } + if c.clear { + // Notify after 500ms, giving the time to receive + // all/most of the changes of a batch update + // TODO parameterize this delay + time.AfterFunc(500*time.Millisecond, func() { c.updateQueue.Notify() }) + } + c.clear = false +} + +// implements converters.types.Cache +func (c *k8scache) NeedResync() bool { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + return c.needResync +} + +// implements converters.types.Cache +func (c *k8scache) GlobalConfig() (cur, new map[string]string) { + return c.globalConfigMapData, c.newGlobalConfigMapData +} + +// implements converters.types.Cache +func (c *k8scache) GetDirtyIngresses() (del, upd, add []*extensions.Ingress) { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + del = make([]*extensions.Ingress, len(c.delIngresses)) + for i := range c.delIngresses { + del[i] = c.delIngresses[i] + } + upd = make([]*extensions.Ingress, len(c.updIngresses)) + for i := range c.updIngresses { + upd[i] = c.updIngresses[i] + } + add = make([]*extensions.Ingress, len(c.addIngresses)) + for i := range c.addIngresses { + add[i] = c.addIngresses[i] + } + return del, upd, add +} + +// implements converters.types.Cache +func (c *k8scache) GetDirtyEndpoints() []*api.Endpoints { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + ep := make([]*api.Endpoints, len(c.newEndpoints)) + for i := range c.newEndpoints { + ep[i] = c.newEndpoints[i] + } + return ep +} + +// implements converters.types.Cache +func (c *k8scache) GetDirtyServices() (del, upd, add []*api.Service) { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + del = make([]*api.Service, len(c.delServices)) + for i := range c.delServices { + del[i] = c.delServices[i] + } + upd = make([]*api.Service, len(c.updServices)) + for i := range c.updServices { + upd[i] = c.updServices[i] + } + add = make([]*api.Service, len(c.addServices)) + for i := range c.addServices { + add[i] = c.addServices[i] + } + return del, upd, add +} + +// implements converters.types.Cache +func (c *k8scache) GetDirtySecrets() (del, upd, add []*api.Secret) { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + del = make([]*api.Secret, len(c.delSecrets)) + for i := range c.delSecrets { + del[i] = c.delSecrets[i] + } + upd = make([]*api.Secret, len(c.updSecrets)) + for i := range c.updSecrets { + upd[i] = c.updSecrets[i] + } + add = make([]*api.Secret, len(c.addSecrets)) + for i := range c.addSecrets { + add[i] = c.addSecrets[i] + } + return del, upd, add +} + +// implements converters.types.Cache +func (c *k8scache) GetDirtyPods() []*api.Pod { + c.stateMutex.RLock() + defer c.stateMutex.RUnlock() + pods := make([]*api.Pod, len(c.newPods)) + for i := range c.newPods { + pods[i] = c.newPods[i] + } + return pods +} + +// implements converters.types.Cache +func (c *k8scache) SyncNewObjects() { + // IMPLEMENT + // lock between the first state reading and this sync + // this will avoid loose unread state change + c.stateMutex.Lock() + defer c.stateMutex.Unlock() + // + c.newPods = nil + c.newEndpoints = nil + // + // Secrets + // + c.delSecrets = nil + c.updSecrets = nil + c.addSecrets = nil + // + // Ingress + // + c.delIngresses = nil + c.updIngresses = nil + c.addIngresses = nil + // + // ConfigMaps + // + if c.newGlobalConfigMapData != nil { + c.globalConfigMapData = c.newGlobalConfigMapData + c.newGlobalConfigMapData = nil + } + if c.newTCPConfigMapData != nil { + c.tcpConfigMapData = c.newTCPConfigMapData + c.newTCPConfigMapData = nil + } + // + c.clear = true + c.needResync = false +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f1135baa0..c82c9b0ae 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -20,18 +20,13 @@ import ( "context" "fmt" "net/http" - "sort" "time" "github.com/golang/glog" "github.com/spf13/pflag" api "k8s.io/api/core/v1" extensions "k8s.io/api/extensions/v1beta1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes/scheme" - typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/tools/record" "github.com/jcmoraisjr/haproxy-ingress/pkg/acme" "github.com/jcmoraisjr/haproxy-ingress/pkg/common/ingress" @@ -39,6 +34,7 @@ import ( "github.com/jcmoraisjr/haproxy-ingress/pkg/common/net/ssl" configmapconverter "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/configmap" ingressconverter "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress" + "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/tracker" ingtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/ingress/types" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" @@ -61,8 +57,6 @@ type HAProxyController struct { controller *controller.GenericController cfg *controller.Configuration configMap *api.ConfigMap - recorder record.EventRecorder - listers *listers converterOptions *ingtypes.ConverterOptions reloadStrategy *string maxOldConfigFiles *int @@ -109,17 +103,8 @@ func (hc *HAProxyController) configController() { if hc.cfg.ForceNamespaceIsolation { watchNamespace = hc.cfg.WatchNamespace } - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(hc.logger.Info) - eventBroadcaster.StartRecordingToSink(&typedv1.EventSinkImpl{ - Interface: hc.cfg.Client.CoreV1().Events(watchNamespace), - }) - hc.recorder = eventBroadcaster.NewRecorder(scheme.Scheme, api.EventSource{ - Component: "ingress-controller", - }) - hc.listers = createListers(hc, hc.logger, hc.recorder, hc.cfg.Client, watchNamespace, hc.cfg.ResyncPeriod) - hc.cache = newCache(hc.cfg.Client, hc.listers, hc.controller) hc.ingressQueue = utils.NewRateLimitingQueue(hc.cfg.RateLimitUpdate, hc.syncIngress) + hc.cache = createCache(hc.logger, hc.cfg.Client, hc.controller, hc.ingressQueue, watchNamespace, hc.cfg.ResyncPeriod) var acmeSigner acme.Signer if hc.cfg.AcmeServer { electorID := fmt.Sprintf("%s-%s", hc.cfg.AcmeElectionID, hc.cfg.IngressClass) @@ -150,6 +135,7 @@ func (hc *HAProxyController) configController() { hc.converterOptions = &ingtypes.ConverterOptions{ Logger: hc.logger, Cache: hc.cache, + Tracker: tracker.NewTracker(), AnnotationPrefix: hc.cfg.AnnPrefix, DefaultBackend: hc.cfg.DefaultService, DefaultSSLFile: hc.createDefaultSSLFile(), @@ -159,7 +145,7 @@ func (hc *HAProxyController) configController() { } func (hc *HAProxyController) startServices() { - hc.listers.RunAsync(hc.stopCh) + hc.cache.RunAsync(hc.stopCh) go hc.ingressQueue.Run() if hc.cfg.StatsCollectProcPeriod.Milliseconds() > 0 { go wait.Until(func() { @@ -258,17 +244,10 @@ func (hc *HAProxyController) Stop() error { return err } -// Notify ... -// implements ListerEvents -// implements oldcontroller.NewCtrlIntf -func (hc *HAProxyController) Notify() { - hc.ingressQueue.Notify() -} - // GetIngressList ... // implements oldcontroller.NewCtrlIntf func (hc *HAProxyController) GetIngressList() ([]*extensions.Ingress, error) { - return hc.listers.ingressLister.List(labels.Everything()) + return hc.cache.GetIngressList() } // GetSecret ... @@ -277,49 +256,6 @@ func (hc *HAProxyController) GetSecret(name string) (*api.Secret, error) { return hc.cache.GetSecret(name) } -// UpdateSecret ... -// implements ListerEvents -func (hc *HAProxyController) UpdateSecret(key string) { - hc.controller.SyncSecret(key) -} - -// DeleteSecret ... -// implements ListerEvents -func (hc *HAProxyController) DeleteSecret(key string) { - hc.controller.DeleteSecret(key) - hc.ingressQueue.Notify() -} - -// AddConfigMap ... -// implements ListerEvents -func (hc *HAProxyController) AddConfigMap(cm *api.ConfigMap) { - key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) - if key == hc.cfg.ConfigMapName { - hc.logger.InfoV(2, "adding configmap %v to backend", key) - hc.configMap = cm - } -} - -// UpdateConfigMap ... -// implements ListerEvents -func (hc *HAProxyController) UpdateConfigMap(cm *api.ConfigMap) { - key := fmt.Sprintf("%s/%s", cm.Namespace, cm.Name) - if key == hc.cfg.ConfigMapName { - hc.logger.InfoV(2, "updating configmap backend (%v)", key) - hc.configMap = cm - } - if key == hc.cfg.ConfigMapName || key == hc.cfg.TCPConfigMapName { - hc.recorder.Eventf(cm, api.EventTypeNormal, "UPDATE", fmt.Sprintf("ConfigMap %v", key)) - hc.ingressQueue.Notify() - } -} - -// IsValidClass ... -// implements ListerEvents -func (hc *HAProxyController) IsValidClass(ing *extensions.Ingress) bool { - return hc.controller.IsValidClass(ing) -} - // Name provides the complete name of the controller func (hc *HAProxyController) Name() string { return "HAProxy Ingress Controller" @@ -381,35 +317,11 @@ func (hc *HAProxyController) syncIngress(item interface{}) { hc.updateCount++ hc.logger.Info("starting HAProxy update id=%d", hc.updateCount) timer := utils.NewTimer(hc.metrics.ControllerProcTime) - var ingress []*extensions.Ingress - il, err := hc.listers.ingressLister.List(labels.Everything()) - if err != nil { - hc.logger.Error("error reading ingress list: %v", err) - return - } - for _, ing := range il { - if hc.controller.IsValidClass(ing) { - ingress = append(ingress, ing) - } - } - sort.Slice(ingress, func(i, j int) bool { - i1 := ingress[i] - i2 := ingress[j] - if i1.CreationTimestamp != i2.CreationTimestamp { - return i1.CreationTimestamp.Before(&i2.CreationTimestamp) - } - return i1.Namespace+"/"+i1.Name < i2.Namespace+"/"+i2.Name - }) - var globalConfig map[string]string - if hc.configMap != nil { - globalConfig = hc.configMap.Data - } ingConverter := ingressconverter.NewIngressConverter( hc.converterOptions, hc.instance.Config(), - globalConfig, ) - ingConverter.Sync(ingress) + ingConverter.Sync() timer.Tick("parse_ingress") // diff --git a/pkg/controller/listers.go b/pkg/controller/listers.go index 7334fbfae..5dc9d66fe 100644 --- a/pkg/controller/listers.go +++ b/pkg/controller/listers.go @@ -39,15 +39,9 @@ import ( // ListerEvents ... type ListerEvents interface { - Notify() - // - UpdateSecret(key string) - DeleteSecret(key string) - // - AddConfigMap(cm *api.ConfigMap) - UpdateConfigMap(cm *api.ConfigMap) - // - IsValidClass(ing *extensions.Ingress) bool + IsValidIngress(ing *extensions.Ingress) bool + IsValidConfigMap(cm *api.ConfigMap) bool + Notify(old, cur interface{}) } type listers struct { @@ -133,8 +127,8 @@ func (l *listers) createIngressLister(informer informersv1beta1.IngressInformer) l.ingressInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { ing := obj.(*extensions.Ingress) - if l.events.IsValidClass(ing) { - l.events.Notify() + if l.events.IsValidIngress(ing) { + l.events.Notify(nil, ing) l.recorder.Eventf(ing, api.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", ing.Namespace, ing.Name)) } }, @@ -144,19 +138,21 @@ func (l *listers) createIngressLister(informer informersv1beta1.IngressInformer) } oldIng := old.(*extensions.Ingress) curIng := cur.(*extensions.Ingress) - oldValid := l.events.IsValidClass(oldIng) - curValid := l.events.IsValidClass(curIng) + oldValid := l.events.IsValidIngress(oldIng) + curValid := l.events.IsValidIngress(curIng) if !oldValid && !curValid { return } if !oldValid && curValid { + l.events.Notify(nil, curIng) l.recorder.Eventf(curIng, api.EventTypeNormal, "CREATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name)) } else if oldValid && !curValid { + l.events.Notify(oldIng, nil) l.recorder.Eventf(curIng, api.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name)) } else { + l.events.Notify(oldIng, curIng) l.recorder.Eventf(curIng, api.EventTypeNormal, "UPDATE", fmt.Sprintf("Ingress %s/%s", curIng.Namespace, curIng.Name)) } - l.events.Notify() }, DeleteFunc: func(obj interface{}) { ing, ok := obj.(*extensions.Ingress) @@ -164,18 +160,20 @@ func (l *listers) createIngressLister(informer informersv1beta1.IngressInformer) tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { l.logger.Error("couldn't get object from tombstone %#v", obj) + l.events.Notify(nil, nil) return } if ing, ok = tombstone.Obj.(*extensions.Ingress); !ok { l.logger.Error("Tombstone contained object that is not an Ingress: %#v", obj) + l.events.Notify(nil, nil) return } } - if !l.events.IsValidClass(ing) { + if !l.events.IsValidIngress(ing) { return } l.recorder.Eventf(ing, api.EventTypeNormal, "DELETE", fmt.Sprintf("Ingress %s/%s", ing.Namespace, ing.Name)) - l.events.Notify() + l.events.Notify(ing, nil) }, }) } @@ -185,17 +183,17 @@ func (l *listers) createEndpointLister(informer informersv1.EndpointsInformer) { l.endpointInformer = informer.Informer() l.endpointInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(nil, obj) }, UpdateFunc: func(old, cur interface{}) { oldEP := old.(*api.Endpoints) curEP := cur.(*api.Endpoints) if !reflect.DeepEqual(oldEP.Subsets, curEP.Subsets) { - l.events.Notify() + l.events.Notify(oldEP, curEP) } }, DeleteFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(obj, nil) }, }) } @@ -203,6 +201,31 @@ func (l *listers) createEndpointLister(informer informersv1.EndpointsInformer) { func (l *listers) createServiceLister(informer informersv1.ServiceInformer) { l.serviceLister = informer.Lister() l.serviceInformer = informer.Informer() + l.serviceInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + l.events.Notify(nil, obj) + }, + UpdateFunc: func(old, cur interface{}) { + if !reflect.DeepEqual(old, cur) { + l.events.Notify(old, cur) + } + }, + DeleteFunc: func(obj interface{}) { + svc, ok := obj.(*api.Service) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + l.logger.Error("couldn't get object from tombstone %#v", obj) + return + } + if svc, ok = tombstone.Obj.(*api.Service); !ok { + l.logger.Error("Tombstone contained object that is not a Service: %#v", obj) + return + } + } + l.events.Notify(svc, nil) + }, + }) } func (l *listers) createSecretLister(informer informersv1.SecretInformer) { @@ -210,13 +233,11 @@ func (l *listers) createSecretLister(informer informersv1.SecretInformer) { l.secretInformer = informer.Informer() l.secretInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(nil, obj) }, UpdateFunc: func(old, cur interface{}) { if !reflect.DeepEqual(old, cur) { - sec := cur.(*api.Secret) - key := fmt.Sprintf("%v/%v", sec.Namespace, sec.Name) - l.events.UpdateSecret(key) + l.events.Notify(old, cur) } }, DeleteFunc: func(obj interface{}) { @@ -232,8 +253,7 @@ func (l *listers) createSecretLister(informer informersv1.SecretInformer) { return } } - key := fmt.Sprintf("%v/%v", sec.Namespace, sec.Name) - l.events.DeleteSecret(key) + l.events.Notify(sec, nil) }, }) } @@ -243,11 +263,15 @@ func (l *listers) createConfigMapLister(informer informersv1.ConfigMapInformer) l.configMapInformer = informer.Informer() l.configMapInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - l.events.AddConfigMap(obj.(*api.ConfigMap)) + if l.events.IsValidConfigMap(obj.(*api.ConfigMap)) { + l.events.Notify(nil, obj) + } }, UpdateFunc: func(old, cur interface{}) { if !reflect.DeepEqual(old, cur) { - l.events.UpdateConfigMap(cur.(*api.ConfigMap)) + if l.events.IsValidConfigMap(cur.(*api.ConfigMap)) { + l.events.Notify(old, cur) + } } }, }) @@ -261,11 +285,11 @@ func (l *listers) createPodLister(informer informersv1.PodInformer) { oldPod := old.(*api.Pod) curPod := cur.(*api.Pod) if oldPod.DeletionTimestamp != curPod.DeletionTimestamp { - l.events.Notify() + l.events.Notify(old, cur) } }, DeleteFunc: func(obj interface{}) { - l.events.Notify() + l.events.Notify(obj, nil) }, }) } diff --git a/pkg/converters/helper_test/cachemock.go b/pkg/converters/helper_test/cachemock.go index 4832a5163..3f4347521 100644 --- a/pkg/converters/helper_test/cachemock.go +++ b/pkg/converters/helper_test/cachemock.go @@ -23,6 +23,7 @@ import ( "time" api "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" ) @@ -32,6 +33,8 @@ type SecretContent map[string]map[string][]byte // CacheMock ... type CacheMock struct { + GlobalCfg map[string]string + IngList []*extensions.Ingress SvcList []*api.Service EpList map[string]*api.Endpoints TermPodList map[string][]*api.Pod @@ -62,6 +65,16 @@ func (c *CacheMock) buildSecretName(defaultNamespace, secretName string) string return defaultNamespace + "/" + secretName } +// GetIngress ... +func (c *CacheMock) GetIngress(ingressName string) (*extensions.Ingress, error) { + return nil, nil +} + +// GetIngressList ... +func (c *CacheMock) GetIngressList() ([]*extensions.Ingress, error) { + return c.IngList, nil +} + // GetService ... func (c *CacheMock) GetService(serviceName string) (*api.Service, error) { sname := strings.Split(serviceName, "/") @@ -158,3 +171,42 @@ func (c *CacheMock) GetSecretContent(defaultNamespace, secretName, keyName strin } return nil, fmt.Errorf("secret not found: '%s'", fullname) } + +// NeedResync ... +func (c *CacheMock) NeedResync() bool { + return true +} + +// GlobalConfig ... +func (c *CacheMock) GlobalConfig() (cur, dirty map[string]string) { + return c.GlobalCfg, nil +} + +// GetDirtyIngresses ... +func (c *CacheMock) GetDirtyIngresses() (del, upd, add []*extensions.Ingress) { + return nil, nil, nil +} + +// GetDirtyEndpoints ... +func (c *CacheMock) GetDirtyEndpoints() []*api.Endpoints { + return nil +} + +// GetDirtyServices ... +func (c *CacheMock) GetDirtyServices() (del, upd, add []*api.Service) { + return nil, nil, nil +} + +// GetDirtySecrets ... +func (c *CacheMock) GetDirtySecrets() (del, upd, add []*api.Secret) { + return nil, nil, nil +} + +// GetDirtyPods ... +func (c *CacheMock) GetDirtyPods() []*api.Pod { + return nil +} + +// SyncNewObjects ... +func (c *CacheMock) SyncNewObjects() { +} diff --git a/pkg/converters/helper_test/trackermock.go b/pkg/converters/helper_test/trackermock.go new file mode 100644 index 000000000..91588ae52 --- /dev/null +++ b/pkg/converters/helper_test/trackermock.go @@ -0,0 +1,55 @@ +/* +Copyright 2020 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helper_test + +import ( + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" +) + +// TrackerMock ... +type TrackerMock struct{} + +// NewTrackerMock ... +func NewTrackerMock() *TrackerMock { + return &TrackerMock{} +} + +// TrackHostname ... +func (t *TrackerMock) TrackHostname(rtype convtypes.ResourceType, name, hostname string) { +} + +// TrackBackend ... +func (t *TrackerMock) TrackBackend(rtype convtypes.ResourceType, name string, backendID hatypes.BackendID) { +} + +// TrackMissingOnHostname ... +func (t *TrackerMock) TrackMissingOnHostname(rtype convtypes.ResourceType, name, hostname string) { +} + +// GetDirtyLinks ... +func (t *TrackerMock) GetDirtyLinks(oldIngList, oldServiceList, addServiceList, oldSecretList, addSecretList []string) (dirtyIngs, dirtyHosts []string, dirtyBacks []hatypes.BackendID) { + return nil, nil, nil +} + +// DeleteHostnames ... +func (t *TrackerMock) DeleteHostnames(hostnames []string) { +} + +// DeleteBackends ... +func (t *TrackerMock) DeleteBackends(backends []hatypes.BackendID) { +} diff --git a/pkg/converters/ingress/annotations/mapper.go b/pkg/converters/ingress/annotations/mapper.go index f29c728e1..29e86c158 100644 --- a/pkg/converters/ingress/annotations/mapper.go +++ b/pkg/converters/ingress/annotations/mapper.go @@ -311,6 +311,11 @@ func (cv *ConfigValue) Int64() int64 { return value } +// FullName ... +func (s *Source) FullName() string { + return s.Namespace + "/" + s.Name +} + // String ... func (m *Map) String() string { return fmt.Sprintf("%+v", *m) @@ -318,5 +323,5 @@ func (m *Map) String() string { // String ... func (s *Source) String() string { - return s.Type + " '" + s.Namespace + "/" + s.Name + "'" + return s.Type + " '" + s.FullName() + "'" } diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index dc020b1bb..210d7b9b3 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -18,6 +18,8 @@ package ingress import ( "fmt" + "reflect" + "sort" "strconv" "strings" @@ -35,15 +37,19 @@ import ( // Config ... type Config interface { - Sync(ingress []*extensions.Ingress) + Sync() } // NewIngressConverter ... -func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Config, globalConfig map[string]string) Config { +func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Config) Config { if options.DefaultConfig == nil { options.DefaultConfig = createDefaults } defaultConfig := options.DefaultConfig() + globalConfig, newConfig := options.Cache.GlobalConfig() + if newConfig != nil { + globalConfig = newConfig + } for key, value := range globalConfig { defaultConfig[key] = value } @@ -52,6 +58,7 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con options: options, logger: options.Logger, cache: options.Cache, + tracker: options.Tracker, mapBuilder: annotations.NewMapBuilder(options.Logger, options.AnnotationPrefix+"/", defaultConfig), updater: annotations.NewUpdater(haproxy, options), globalConfig: annotations.NewMapBuilder(options.Logger, "", defaultConfig).NewMapper(), @@ -60,7 +67,7 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con } haproxy.Frontend().DefaultCert = options.DefaultSSLFile.Filename if options.DefaultBackend != "" { - if backend, err := c.addBackend(&annotations.Source{}, "*/", options.DefaultBackend, "", map[string]string{}); err == nil { + if backend, err := c.addBackend(&annotations.Source{}, "*", "/", options.DefaultBackend, "", map[string]string{}); err == nil { haproxy.Backends().SetDefaultBackend(backend) } else { c.logger.Error("error reading default service: %v", err) @@ -74,6 +81,7 @@ type converter struct { options *ingtypes.ConverterOptions logger types.Logger cache convtypes.Cache + tracker convtypes.Tracker mapBuilder *annotations.MapBuilder updater annotations.Updater globalConfig *annotations.Mapper @@ -81,13 +89,162 @@ type converter struct { backendAnnotations map[*hatypes.Backend]*annotations.Mapper } -func (c *converter) Sync(ingress []*extensions.Ingress) { - for _, ing := range ingress { - c.syncIngress(ing) +func (c *converter) Sync() { + globalConfig, newConfig := c.cache.GlobalConfig() + // IMPLEMENT + // config option to allow partial parsing + // cache also need to know if partial parsing is enabled + needResync := true || c.cache.NeedResync() || globalConfigNeedFullSync(globalConfig, newConfig) + if needResync { + c.syncFull() + } else { + c.syncPartial() } c.syncAnnotations() } +func globalConfigNeedFullSync(cur, new map[string]string) bool { + // Currently if a global is changed, all the ingress objects are parsed again. + // This need to be done due to: + // + // 1. Default host and backend annotations. If a default value + // changes, such default may impact any ingress object; + // 2. At the time of this writing, the following global + // configuration keys are used during annotation parsing: + // * GlobalDNSResolvers + // * GlobalDrainSupport + // * GlobalNoTLSRedirectLocations + // + // This might be improved after implement a way to guarantee that a global + // is just a haproxy global, default or frontend config. + return new != nil && !reflect.DeepEqual(cur, new) +} + +func (c *converter) syncFull() { + c.haproxy.Clear() + ingList, err := c.cache.GetIngressList() + if err != nil { + c.logger.Error("error reading ingress list: %v", err) + return + } + sortIngress(ingList) + for _, ing := range ingList { + c.syncIngress(ing) + } +} + +func (c *converter) syncPartial() { + // conventions: + // + // * del, upd, add: events from the listers + // * old, new: old state (deleted, before change) and new state (after change, added) + // * dirty: has an impact due to a direct or indirect change + // + + // helper funcs + ing2names := func(ings []*extensions.Ingress) []string { + inglist := make([]string, len(ings)) + for i, ing := range ings { + inglist[i] = ing.Namespace + "/" + ing.Name + } + return inglist + } + svc2names := func(services []*api.Service) []string { + serviceList := make([]string, len(services)) + for i, service := range services { + serviceList[i] = service.Namespace + "/" + service.Name + } + return serviceList + } + secret2names := func(secrets []*api.Secret) []string { + secretList := make([]string, len(secrets)) + for i, secret := range secrets { + secretList[i] = secret.Namespace + "/" + secret.Name + } + return secretList + } + + // changed objects + delIngs, updIngs, addIngs := c.cache.GetDirtyIngresses() + endpoints := c.cache.GetDirtyEndpoints() + delSvcs, updSvcs, addSvcs := c.cache.GetDirtyServices() + delSecrets, updSecrets, addSecrets := c.cache.GetDirtySecrets() + pods := c.cache.GetDirtyPods() + c.cache.SyncNewObjects() + + // remove changed/deleted data + delIngNames := ing2names(delIngs) + updIngNames := ing2names(updIngs) + oldIngNames := append(delIngNames, updIngNames...) + delSvcNames := svc2names(delSvcs) + updSvcNames := svc2names(updSvcs) + addSvcNames := svc2names(addSvcs) + oldSvcNames := append(delSvcNames, updSvcNames...) + delSecretNames := secret2names(delSecrets) + updSecretNames := secret2names(updSecrets) + addSecretNames := secret2names(addSecrets) + oldSecretNames := append(delSecretNames, updSecretNames...) + dirtyIngs, dirtyHosts, dirtyBacks := + c.tracker.GetDirtyLinks(oldIngNames, oldSvcNames, addSvcNames, oldSecretNames, addSecretNames) + c.haproxy.Hosts().RemoveAll(dirtyHosts) + c.haproxy.Backends().RemoveAll(dirtyBacks) + c.tracker.DeleteHostnames(dirtyHosts) + c.tracker.DeleteBackends(dirtyBacks) + + // merge dirty and added ingress objects into a single list + ingMap := make(map[string]*extensions.Ingress) + for _, ing := range dirtyIngs { + ingMap[ing] = nil + } + for _, ing := range delIngNames { + delete(ingMap, ing) + } + for _, ing := range addIngs { + ingMap[ing.Namespace+"/"+ing.Name] = ing + } + ingList := make([]*extensions.Ingress, 0, len(ingMap)) + for name, ing := range ingMap { + if ing == nil { + var err error + ing, err = c.cache.GetIngress(name) + if err != nil { + c.logger.Warn("ignoring ingress '%s': %v", name, err) + ing = nil + } + } + if ing != nil { + ingList = append(ingList, ing) + } + } + + // reinclude changed/added data + sortIngress(ingList) + for _, ing := range ingList { + c.syncIngress(ing) + } + for _, ep := range endpoints { + if err := c.applyEndpoints(ep); err != nil { + c.logger.Warn("skipping apply endpoint '%s/%s' update: %v", ep.Namespace, ep.Name, err) + } + } + if c.globalConfig.Get(ingtypes.GlobalDrainSupport).Bool() { + for _, pod := range pods { + c.applyPod(pod) + } + } +} + +func sortIngress(ingress []*extensions.Ingress) { + sort.Slice(ingress, func(i, j int) bool { + i1 := ingress[i] + i2 := ingress[j] + if i1.CreationTimestamp != i2.CreationTimestamp { + return i1.CreationTimestamp.Before(&i2.CreationTimestamp) + } + return i1.Namespace+"/"+i1.Name < i2.Namespace+"/"+i2.Name + }) +} + func (c *converter) syncIngress(ing *extensions.Ingress) { fullIngName := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) source := &annotations.Source{ @@ -123,7 +280,7 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { } svcName, svcPort := readServiceNamePort(&path.Backend) fullSvcName := ing.Namespace + "/" + svcName - backend, err := c.addBackend(source, hostname+uri, fullSvcName, svcPort, annBack) + backend, err := c.addBackend(source, hostname, uri, fullSvcName, svcPort, annBack) if err != nil { c.logger.Warn("skipping backend config of ingress '%s': %v", fullIngName, err) continue @@ -132,7 +289,7 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { sslpassthrough, _ := strconv.ParseBool(annHost[ingtypes.HostSSLPassthrough]) sslpasshttpport := annHost[ingtypes.HostSSLPassthroughHTTPPort] if sslpassthrough && sslpasshttpport != "" { - if _, err := c.addBackend(source, hostname+uri, fullSvcName, sslpasshttpport, annBack); err != nil { + if _, err := c.addBackend(source, hostname, uri, fullSvcName, sslpasshttpport, annBack); err != nil { c.logger.Warn("skipping http port config of ssl-passthrough on %v: %v", source, err) } } @@ -140,7 +297,7 @@ func (c *converter) syncIngress(ing *extensions.Ingress) { for _, tls := range ing.Spec.TLS { for _, tlshost := range tls.Hosts { if tlshost == hostname { - tlsPath := c.addTLS(source, tls.SecretName) + tlsPath := c.addTLS(source, tlshost, tls.SecretName) if host.TLS.TLSHash == "" { host.TLS.TLSFilename = tlsPath.Filename host.TLS.TLSHash = tlsPath.SHA1Hash @@ -200,7 +357,7 @@ func (c *converter) addDefaultHostBackend(source *annotations.Source, fullSvcNam return fmt.Errorf("path %s was already defined on default host", uri) } } - backend, err := c.addBackend(source, hostname+uri, fullSvcName, svcPort, annBack) + backend, err := c.addBackend(source, hostname, uri, fullSvcName, svcPort, annBack) if err != nil { return err } @@ -210,7 +367,9 @@ func (c *converter) addDefaultHostBackend(source *annotations.Source, fullSvcNam } func (c *converter) addHost(hostname string, source *annotations.Source, ann map[string]string) *hatypes.Host { + // TODO build a stronger tracking host := c.haproxy.Hosts().AcquireHost(hostname) + c.tracker.TrackHostname(convtypes.IngressType, source.FullName(), hostname) mapper, found := c.hostAnnotations[host] if !found { mapper = c.mapBuilder.NewMapper() @@ -223,11 +382,14 @@ func (c *converter) addHost(hostname string, source *annotations.Source, ann map return host } -func (c *converter) addBackend(source *annotations.Source, hostpath, fullSvcName, svcPort string, ann map[string]string) (*hatypes.Backend, error) { +func (c *converter) addBackend(source *annotations.Source, hostname, uri, fullSvcName, svcPort string, ann map[string]string) (*hatypes.Backend, error) { + // TODO build a stronger tracking svc, err := c.cache.GetService(fullSvcName) if err != nil { + c.tracker.TrackMissingOnHostname(convtypes.ServiceType, fullSvcName, hostname) return nil, err } + c.tracker.TrackHostname(convtypes.ServiceType, fullSvcName, hostname) ssvcName := strings.Split(fullSvcName, "/") namespace := ssvcName[0] svcName := ssvcName[1] @@ -241,6 +403,8 @@ func (c *converter) addBackend(source *annotations.Source, hostpath, fullSvcName return nil, fmt.Errorf("port not found: '%s'", svcPort) } backend := c.haproxy.Backends().AcquireBackend(namespace, svcName, port.TargetPort.String()) + c.tracker.TrackBackend(convtypes.IngressType, source.FullName(), backend.BackendID()) + hostpath := hostname + uri mapper, found := c.backendAnnotations[backend] if !found { // New backend, initialize with service annotations, giving precedence @@ -285,12 +449,14 @@ func (c *converter) addBackend(source *annotations.Source, hostpath, fullSvcName return backend, nil } -func (c *converter) addTLS(source *annotations.Source, secretName string) convtypes.CrtFile { +func (c *converter) addTLS(source *annotations.Source, hostname, secretName string) convtypes.CrtFile { if secretName != "" { tlsFile, err := c.cache.GetTLSSecretPath(source.Namespace, secretName) if err == nil { + c.tracker.TrackHostname(convtypes.SecretType, secretName, hostname) return tlsFile } + c.tracker.TrackMissingOnHostname(convtypes.SecretType, secretName, hostname) c.logger.Warn("using default certificate due to an error reading secret '%s' on %s: %v", secretName, source, err) } return c.options.DefaultSSLFile @@ -327,6 +493,27 @@ func (c *converter) addEndpoints(svc *api.Service, svcPort *api.ServicePort, bac return nil } +func (c *converter) applyEndpoints(endpoints *api.Endpoints) error { + svc, err := c.cache.GetService(fmt.Sprintf("%s/%s", endpoints.Namespace, endpoints.Name)) + if err != nil { + return err + } + for _, port := range svc.Spec.Ports { + backend := c.haproxy.Backends().FindBackend(endpoints.Namespace, endpoints.Name, port.TargetPort.String()) + if backend != nil { + backend.ClearEndpoints() + if err := c.addEndpoints(svc, &port, backend); err != nil { + c.logger.Warn("skipping backend '%s' update: %v", backend.ID, err) + } + } + } + return nil +} + +func (c *converter) applyPod(pod *api.Pod) { + // IMPLEMENT +} + func (c *converter) readAnnotations(annotations map[string]string) (annHost, annBack map[string]string) { annHost = make(map[string]string, len(annotations)) annBack = make(map[string]string, len(annotations)) diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index 121dc1109..1d12fd0c2 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -1107,6 +1107,7 @@ type testConfig struct { hconfig haproxy.Config logger *types_helper.LoggerMock cache *conv_helper.CacheMock + tracker *conv_helper.TrackerMock updater *updaterMock } @@ -1118,6 +1119,7 @@ func setup(t *testing.T) *testConfig { hconfig: haproxy.CreateInstance(logger, haproxy.InstanceOptions{}).Config(), cache: conv_helper.NewCacheMock(), logger: logger, + tracker: conv_helper.NewTrackerMock(), } c.createSvc1("system/default", "8080", "172.17.0.99") return c @@ -1138,6 +1140,8 @@ var defaultBackendConfig = ` port: 8080` func (c *testConfig) SyncDef(config map[string]string, ing ...*extensions.Ingress) { + c.cache.IngList = ing + c.cache.GlobalCfg = config defaultConfig := func() map[string]string { return map[string]string{ ingtypes.BackInitialWeight: "100", @@ -1147,6 +1151,7 @@ func (c *testConfig) SyncDef(config map[string]string, ing ...*extensions.Ingres &ingtypes.ConverterOptions{ Cache: c.cache, Logger: c.logger, + Tracker: c.tracker, DefaultConfig: defaultConfig, DefaultBackend: "system/default", DefaultSSLFile: convtypes.CrtFile{ @@ -1158,10 +1163,9 @@ func (c *testConfig) SyncDef(config map[string]string, ing ...*extensions.Ingres AnnotationPrefix: "ingress.kubernetes.io", }, c.hconfig, - config, ).(*converter) conv.updater = c.updater - conv.Sync(ing) + conv.Sync() } func (c *testConfig) createSvc1Auto() (*api.Service, *api.Endpoints) { diff --git a/pkg/converters/ingress/tracker/tracker.go b/pkg/converters/ingress/tracker/tracker.go new file mode 100644 index 000000000..f26550df8 --- /dev/null +++ b/pkg/converters/ingress/tracker/tracker.go @@ -0,0 +1,379 @@ +/* +Copyright 2020 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tracker + +import ( + "fmt" + + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" +) + +// NewTracker ... +func NewTracker() convtypes.Tracker { + return &tracker{} +} + +type ( + stringStringMap map[string]map[string]empty + stringBackendMap map[string]map[hatypes.BackendID]empty + backendStringMap map[hatypes.BackendID]map[string]empty + // + empty struct{} +) + +type tracker struct { + ingressHostname stringStringMap + hostnameIngress stringStringMap + ingressBackend stringBackendMap + backendIngress backendStringMap + // + serviceHostname stringStringMap + hostnameService stringStringMap + // + secretHostname stringStringMap + hostnameSecret stringStringMap + // + serviceHostnameMissing stringStringMap + hostnameServiceMissing stringStringMap + // + secretHostnameMissing stringStringMap + hostnameSecretMissing stringStringMap +} + +func (t *tracker) TrackHostname(rtype convtypes.ResourceType, name, hostname string) { + switch rtype { + case convtypes.IngressType: + addStringTracking(&t.ingressHostname, name, hostname) + addStringTracking(&t.hostnameIngress, hostname, name) + case convtypes.ServiceType: + addStringTracking(&t.serviceHostname, name, hostname) + addStringTracking(&t.hostnameService, hostname, name) + case convtypes.SecretType: + addStringTracking(&t.secretHostname, name, hostname) + addStringTracking(&t.hostnameSecret, hostname, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func (t *tracker) TrackBackend(rtype convtypes.ResourceType, name string, backendID hatypes.BackendID) { + switch rtype { + case convtypes.IngressType: + addStringBackendTracking(&t.ingressBackend, name, backendID) + addBackendStringTracking(&t.backendIngress, backendID, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +func (t *tracker) TrackMissingOnHostname(rtype convtypes.ResourceType, name, hostname string) { + switch rtype { + case convtypes.ServiceType: + addStringTracking(&t.serviceHostnameMissing, name, hostname) + addStringTracking(&t.hostnameServiceMissing, hostname, name) + case convtypes.SecretType: + addStringTracking(&t.secretHostnameMissing, name, hostname) + addStringTracking(&t.hostnameSecretMissing, hostname, name) + default: + panic(fmt.Errorf("unsupported resource type %d", rtype)) + } +} + +// GetDirtyLinks lists all hostnames and backendIDs that a +// list of ingress touches directly or indirectly: +// +// * when a hostname is listed, all other hostnames of all ingress that +// references it should also be listed; +// * when a backendID (service+port) is listed, all other backendIDs of +// all ingress that references it should also be listed. +// +func (t *tracker) GetDirtyLinks( + oldIngressList []string, + oldServiceList, addServiceList []string, + oldSecretList, addSecretList []string, +) (dirtyIngs, dirtyHosts []string, dirtyBacks []hatypes.BackendID) { + ingsMap := make(map[string]empty) + hostsMap := make(map[string]empty) + backsMap := make(map[hatypes.BackendID]empty) + + // recursively fill hostsMap and backsMap from ingress and secrets + // that directly or indirectly are referenced by them + var build func([]string) + build = func(ingNames []string) { + for _, ingName := range ingNames { + ingsMap[ingName] = empty{} + for _, hostname := range t.getHostnamesByIngress(ingName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + for _, backend := range t.getBackendsByIngress(ingName) { + if _, found := backsMap[backend]; !found { + backsMap[backend] = empty{} + build(t.getIngressByBackend(backend)) + } + } + } + } + build(oldIngressList) + // + for _, svcName := range oldServiceList { + for _, hostname := range t.getHostnamesByService(svcName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + } + for _, svcName := range addServiceList { + for _, hostname := range t.getHostnamesByServiceMissing(svcName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + } + // + for _, secretName := range oldSecretList { + for _, hostname := range t.getHostnamesBySecret(secretName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + } + for _, secretName := range addSecretList { + for _, hostname := range t.getHostnamesBySecretMissing(secretName) { + if _, found := hostsMap[hostname]; !found { + hostsMap[hostname] = empty{} + build(t.getIngressByHostname(hostname)) + } + } + } + + // convert hostsMap and backsMap to slices + if len(ingsMap) > 0 { + dirtyIngs = make([]string, 0, len(ingsMap)) + for ing := range ingsMap { + dirtyIngs = append(dirtyIngs, ing) + } + } + if len(hostsMap) > 0 { + dirtyHosts = make([]string, 0, len(hostsMap)) + for host := range hostsMap { + dirtyHosts = append(dirtyHosts, host) + } + } + if len(backsMap) > 0 { + dirtyBacks = make([]hatypes.BackendID, 0, len(backsMap)) + for back := range backsMap { + dirtyBacks = append(dirtyBacks, back) + } + } + return dirtyIngs, dirtyHosts, dirtyBacks +} + +func (t *tracker) DeleteHostnames(hostnames []string) { + for _, hostname := range hostnames { + for ing := range t.hostnameIngress[hostname] { + deleteStringTracking(&t.ingressHostname, ing, hostname) + } + deleteStringMapKey(&t.hostnameIngress, hostname) + for service := range t.hostnameService[hostname] { + deleteStringTracking(&t.serviceHostname, service, hostname) + } + deleteStringMapKey(&t.hostnameService, hostname) + for service := range t.hostnameServiceMissing[hostname] { + deleteStringTracking(&t.serviceHostnameMissing, service, hostname) + } + deleteStringMapKey(&t.hostnameServiceMissing, hostname) + for secret := range t.hostnameSecret[hostname] { + deleteStringTracking(&t.secretHostname, secret, hostname) + } + deleteStringMapKey(&t.hostnameSecret, hostname) + for secret := range t.hostnameSecretMissing[hostname] { + deleteStringTracking(&t.secretHostnameMissing, secret, hostname) + } + deleteStringMapKey(&t.hostnameSecretMissing, hostname) + } +} + +func (t *tracker) DeleteBackends(backends []hatypes.BackendID) { + for _, backend := range backends { + for ing := range t.backendIngress[backend] { + deleteStringBackendTracking(&t.ingressBackend, ing, backend) + } + deleteBackendStringMapKey(&t.backendIngress, backend) + } +} + +func (t *tracker) getIngressByHostname(hostname string) []string { + if t.hostnameIngress == nil { + return nil + } + return getStringTracking(t.hostnameIngress[hostname]) +} + +func (t *tracker) getHostnamesByIngress(ingName string) []string { + if t.ingressHostname == nil { + return nil + } + return getStringTracking(t.ingressHostname[ingName]) +} + +func (t *tracker) getIngressByBackend(backendID hatypes.BackendID) []string { + if t.backendIngress == nil { + return nil + } + return getStringTracking(t.backendIngress[backendID]) +} + +func (t *tracker) getBackendsByIngress(ingName string) []hatypes.BackendID { + if t.ingressBackend == nil { + return nil + } + return getBackendTracking(t.ingressBackend[ingName]) +} + +func (t *tracker) getHostnamesByService(serviceName string) []string { + if t.serviceHostname == nil { + return nil + } + return getStringTracking(t.serviceHostname[serviceName]) +} + +func (t *tracker) getHostnamesByServiceMissing(serviceName string) []string { + if t.serviceHostnameMissing == nil { + return nil + } + return getStringTracking(t.serviceHostnameMissing[serviceName]) +} + +func (t *tracker) getHostnamesBySecret(secretName string) []string { + if t.secretHostname == nil { + return nil + } + return getStringTracking(t.secretHostname[secretName]) +} + +func (t *tracker) getHostnamesBySecretMissing(secretName string) []string { + if t.secretHostnameMissing == nil { + return nil + } + return getStringTracking(t.secretHostnameMissing[secretName]) +} + +func addStringTracking(trackingRef *stringStringMap, key, value string) { + if *trackingRef == nil { + *trackingRef = stringStringMap{} + } + tracking := *trackingRef + trackingMap, found := tracking[key] + if !found { + trackingMap = map[string]empty{} + tracking[key] = trackingMap + } + trackingMap[value] = empty{} +} + +func addBackendStringTracking(trackingRef *backendStringMap, key hatypes.BackendID, value string) { + if *trackingRef == nil { + *trackingRef = backendStringMap{} + } + tracking := *trackingRef + trackingMap, found := tracking[key] + if !found { + trackingMap = map[string]empty{} + tracking[key] = trackingMap + } + trackingMap[value] = empty{} +} + +func addStringBackendTracking(trackingRef *stringBackendMap, key string, value hatypes.BackendID) { + if *trackingRef == nil { + *trackingRef = stringBackendMap{} + } + tracking := *trackingRef + trackingMap, found := tracking[key] + if !found { + trackingMap = map[hatypes.BackendID]empty{} + tracking[key] = trackingMap + } + trackingMap[value] = empty{} +} + +func getStringTracking(tracking map[string]empty) []string { + stringList := make([]string, 0, len(tracking)) + for value := range tracking { + stringList = append(stringList, value) + } + return stringList +} + +func getBackendTracking(tracking map[hatypes.BackendID]empty) []hatypes.BackendID { + backendList := make([]hatypes.BackendID, 0, len(tracking)) + for value := range tracking { + backendList = append(backendList, value) + } + return backendList +} + +func deleteStringTracking(trackingRef *stringStringMap, key, value string) { + if *trackingRef == nil { + return + } + tracking := *trackingRef + trackingMap := tracking[key] + delete(trackingMap, value) + if len(trackingMap) == 0 { + delete(tracking, key) + } + if len(tracking) == 0 { + *trackingRef = nil + } +} + +func deleteStringBackendTracking(trackingRef *stringBackendMap, key string, value hatypes.BackendID) { + if *trackingRef == nil { + return + } + tracking := *trackingRef + trackingMap := tracking[key] + delete(trackingMap, value) + if len(trackingMap) == 0 { + delete(tracking, key) + } + if len(tracking) == 0 { + *trackingRef = nil + } +} + +func deleteStringMapKey(stringMap *stringStringMap, key string) { + delete(*stringMap, key) + if len(*stringMap) == 0 { + *stringMap = nil + } +} + +func deleteBackendStringMapKey(backendMap *backendStringMap, key hatypes.BackendID) { + delete(*backendMap, key) + if len(*backendMap) == 0 { + *backendMap = nil + } +} diff --git a/pkg/converters/ingress/tracker/tracker_test.go b/pkg/converters/ingress/tracker/tracker_test.go new file mode 100644 index 000000000..01adf1ea7 --- /dev/null +++ b/pkg/converters/ingress/tracker/tracker_test.go @@ -0,0 +1,466 @@ +/* +Copyright 2020 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tracker + +import ( + "reflect" + "sort" + "testing" + + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" +) + +type hostTracking struct { + rtype convtypes.ResourceType + name string + hostname string +} + +type backTracking struct { + rtype convtypes.ResourceType + name string + backend hatypes.BackendID +} + +var ( + back1a = hatypes.BackendID{ + Namespace: "default", + Name: "svc1", + Port: "8080", + } + back1b = hatypes.BackendID{ + Namespace: "default", + Name: "svc1", + Port: "8080", + } + back2a = hatypes.BackendID{ + Namespace: "default", + Name: "svc2", + Port: "8080", + } + back2b = hatypes.BackendID{ + Namespace: "default", + Name: "svc2", + Port: "8080", + } +) + +func TestGetDirtyLinks(t *testing.T) { + testCases := []struct { + trackedHosts []hostTracking + trackedBacks []backTracking + // + trackedMissingHosts []hostTracking + // + oldIngressList []string + oldServiceList []string + addServiceList []string + oldSecretList []string + addSecretList []string + // + expDirtyIngs []string + expDirtyHosts []string + expDirtyBacks []hatypes.BackendID + }{ + // 0 + {}, + // 1 + { + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + }, + // 2 + { + oldServiceList: []string{"default/svc1"}, + }, + // 3 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + }, + // 4 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 5 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + oldServiceList: []string{"default/svc1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 6 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + oldSecretList: []string{"default/secret1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 7 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + trackedMissingHosts: []hostTracking{ + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + addServiceList: []string{"default/svc1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 8 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + trackedMissingHosts: []hostTracking{ + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + addSecretList: []string{"default/secret1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 9 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 10 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + {convtypes.IngressType, "default/ing3", "domain2.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1", "default/ing2"}, + expDirtyHosts: []string{"domain1.local"}, + }, + // 11 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + {convtypes.IngressType, "default/ing3", "domain2.local"}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1", "default/ing2", "default/ing3"}, + expDirtyHosts: []string{"domain1.local", "domain2.local"}, + }, + // 12 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + }, + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1"}, + expDirtyHosts: []string{"domain1.local"}, + expDirtyBacks: []hatypes.BackendID{back1b}, + }, + // 13 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain2.local"}, + {convtypes.IngressType, "default/ing3", "domain3.local"}, + }, + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + {convtypes.IngressType, "default/ing2", back2a}, + {convtypes.IngressType, "default/ing3", back1b}, + }, + oldIngressList: []string{"default/ing1"}, + expDirtyIngs: []string{"default/ing1", "default/ing3"}, + expDirtyHosts: []string{"domain1.local", "domain3.local"}, + expDirtyBacks: []hatypes.BackendID{back1b}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedHost := range test.trackedHosts { + c.tracker.TrackHostname(trackedHost.rtype, trackedHost.name, trackedHost.hostname) + } + for _, trackedBack := range test.trackedBacks { + c.tracker.TrackBackend(trackedBack.rtype, trackedBack.name, trackedBack.backend) + } + for _, trackedMissingHost := range test.trackedMissingHosts { + c.tracker.TrackMissingOnHostname(trackedMissingHost.rtype, trackedMissingHost.name, trackedMissingHost.hostname) + } + dirtyIngs, dirtyHosts, dirtyBacks := + c.tracker.GetDirtyLinks( + test.oldIngressList, + test.oldServiceList, + test.addServiceList, + test.oldSecretList, + test.addSecretList, + ) + sort.Strings(dirtyIngs) + sort.Strings(dirtyHosts) + sort.Slice(dirtyBacks, func(i, j int) bool { + return dirtyBacks[i].String() < dirtyBacks[j].String() + }) + c.compareObjects("dirty ingress", i, dirtyIngs, test.expDirtyIngs) + c.compareObjects("dirty hosts", i, dirtyHosts, test.expDirtyHosts) + c.compareObjects("dirty backs", i, dirtyBacks, test.expDirtyBacks) + c.teardown() + } +} + +func TestDeleteHostnames(t *testing.T) { + testCases := []struct { + trackedHosts []hostTracking + // + trackedMissingHosts []hostTracking + // + deleteHostnames []string + // + expIngressHostname stringStringMap + expHostnameIngress stringStringMap + expServiceHostname stringStringMap + expHostnameService stringStringMap + expSecretHostname stringStringMap + expHostnameSecret stringStringMap + // + expServiceHostnameMissing stringStringMap + expHostnameServiceMissing stringStringMap + expSecretHostnameMissing stringStringMap + expHostnameSecretMissing stringStringMap + }{ + // 0 + {}, + // 1 + { + deleteHostnames: []string{"domain1.local"}, + }, + // 2 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + expIngressHostname: stringStringMap{"default/ing1": {"domain1.local": empty{}}}, + expHostnameIngress: stringStringMap{"domain1.local": {"default/ing1": empty{}}}, + }, + // 3 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 4 + { + trackedHosts: []hostTracking{ + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 5 + { + trackedMissingHosts: []hostTracking{ + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 6 + { + trackedHosts: []hostTracking{ + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 7 + { + trackedMissingHosts: []hostTracking{ + {convtypes.SecretType, "default/secret1", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 8 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing1", "domain2.local"}, + }, + deleteHostnames: []string{"domain1.local", "domain2.local"}, + }, + // 9 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing1", "domain2.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + expIngressHostname: stringStringMap{"default/ing1": {"domain2.local": empty{}}}, + expHostnameIngress: stringStringMap{"domain2.local": {"default/ing1": empty{}}}, + }, + // 10 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local"}, + }, + // 11 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing2", "domain1.local"}, + }, + deleteHostnames: []string{"domain1.local", "domain2.local"}, + }, + // 12 + { + trackedHosts: []hostTracking{ + {convtypes.IngressType, "default/ing1", "domain1.local"}, + {convtypes.IngressType, "default/ing1", "domain2.local"}, + {convtypes.IngressType, "default/ing1", "domain3.local"}, + {convtypes.ServiceType, "default/svc1", "domain1.local"}, + {convtypes.ServiceType, "default/svc1", "domain2.local"}, + {convtypes.ServiceType, "default/svc1", "domain3.local"}, + {convtypes.SecretType, "default/secret1", "domain1.local"}, + {convtypes.SecretType, "default/secret1", "domain2.local"}, + {convtypes.SecretType, "default/secret1", "domain3.local"}, + }, + deleteHostnames: []string{"domain1.local", "domain2.local"}, + expIngressHostname: stringStringMap{"default/ing1": {"domain3.local": empty{}}}, + expHostnameIngress: stringStringMap{"domain3.local": {"default/ing1": empty{}}}, + expServiceHostname: stringStringMap{"default/svc1": {"domain3.local": empty{}}}, + expHostnameService: stringStringMap{"domain3.local": {"default/svc1": empty{}}}, + expSecretHostname: stringStringMap{"default/secret1": {"domain3.local": empty{}}}, + expHostnameSecret: stringStringMap{"domain3.local": {"default/secret1": empty{}}}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedHost := range test.trackedHosts { + c.tracker.TrackHostname(trackedHost.rtype, trackedHost.name, trackedHost.hostname) + } + for _, trackedMissingHost := range test.trackedMissingHosts { + c.tracker.TrackMissingOnHostname(trackedMissingHost.rtype, trackedMissingHost.name, trackedMissingHost.hostname) + } + c.tracker.DeleteHostnames(test.deleteHostnames) + c.compareObjects("ingressHostname", i, c.tracker.ingressHostname, test.expIngressHostname) + c.compareObjects("hostnameIngress", i, c.tracker.hostnameIngress, test.expHostnameIngress) + c.compareObjects("serviceHostname", i, c.tracker.serviceHostname, test.expServiceHostname) + c.compareObjects("hostnameService", i, c.tracker.hostnameService, test.expHostnameService) + c.compareObjects("secretHostname", i, c.tracker.secretHostname, test.expSecretHostname) + c.compareObjects("hostnameSecret", i, c.tracker.hostnameSecret, test.expHostnameSecret) + c.compareObjects("serviceHostnameMissing", i, c.tracker.serviceHostnameMissing, test.expServiceHostnameMissing) + c.compareObjects("hostnameServiceMissing", i, c.tracker.hostnameServiceMissing, test.expHostnameServiceMissing) + c.compareObjects("secretHostnameMissing", i, c.tracker.secretHostnameMissing, test.expSecretHostnameMissing) + c.compareObjects("hostnameSecretMissing", i, c.tracker.hostnameSecretMissing, test.expHostnameSecretMissing) + c.teardown() + } +} + +func TestDeleteBackends(t *testing.T) { + testCases := []struct { + trackedBacks []backTracking + // + deleteBackends []hatypes.BackendID + // + expIngressBackend stringBackendMap + expBackendIngress backendStringMap + }{ + // 0 + {}, + // 1 + { + deleteBackends: []hatypes.BackendID{back1b}, + }, + // 2 + { + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + }, + expBackendIngress: backendStringMap{back1b: {"default/ing1": empty{}}}, + expIngressBackend: stringBackendMap{"default/ing1": {back1b: empty{}}}, + }, + // 3 + { + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + }, + deleteBackends: []hatypes.BackendID{back1b}, + }, + // 4 + { + trackedBacks: []backTracking{ + {convtypes.IngressType, "default/ing1", back1a}, + {convtypes.IngressType, "default/ing2", back1a}, + {convtypes.IngressType, "default/ing2", back2a}, + }, + deleteBackends: []hatypes.BackendID{back1b}, + expBackendIngress: backendStringMap{back2b: {"default/ing2": empty{}}}, + expIngressBackend: stringBackendMap{"default/ing2": {back2b: empty{}}}, + }, + } + for i, test := range testCases { + c := setup(t) + for _, trackedBack := range test.trackedBacks { + c.tracker.TrackBackend(trackedBack.rtype, trackedBack.name, trackedBack.backend) + } + c.tracker.DeleteBackends(test.deleteBackends) + c.compareObjects("ingressBackend", i, c.tracker.ingressBackend, test.expIngressBackend) + c.compareObjects("backendIngress", i, c.tracker.backendIngress, test.expBackendIngress) + c.teardown() + } +} + +type testConfig struct { + t *testing.T + tracker *tracker +} + +func setup(t *testing.T) *testConfig { + return &testConfig{ + t: t, + tracker: NewTracker().(*tracker), + } +} + +func (c *testConfig) teardown() {} + +func (c *testConfig) compareObjects(name string, index int, actual, expected interface{}) { + if !reflect.DeepEqual(actual, expected) { + c.t.Errorf("%s on %d differs - expected: %v - actual: %v", name, index, expected, actual) + } +} diff --git a/pkg/converters/ingress/types/options.go b/pkg/converters/ingress/types/options.go index 60ee307eb..5f2a51d64 100644 --- a/pkg/converters/ingress/types/options.go +++ b/pkg/converters/ingress/types/options.go @@ -25,6 +25,7 @@ import ( type ConverterOptions struct { Logger types.Logger Cache convtypes.Cache + Tracker convtypes.Tracker DefaultConfig func() map[string]string DefaultBackend string DefaultSSLFile convtypes.CrtFile diff --git a/pkg/converters/types/interfaces.go b/pkg/converters/types/interfaces.go index 61f81a996..19c8c9def 100644 --- a/pkg/converters/types/interfaces.go +++ b/pkg/converters/types/interfaces.go @@ -20,10 +20,15 @@ import ( "time" api "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" ) // Cache ... type Cache interface { + GetIngress(ingressName string) (*extensions.Ingress, error) + GetIngressList() ([]*extensions.Ingress, error) GetService(serviceName string) (*api.Service, error) GetEndpoints(service *api.Service) (*api.Endpoints, error) GetTerminatingPods(service *api.Service) ([]*api.Pod, error) @@ -32,6 +37,25 @@ type Cache interface { GetCASecretPath(defaultNamespace, secretName string) (ca, crl File, err error) GetDHSecretPath(defaultNamespace, secretName string) (File, error) GetSecretContent(defaultNamespace, secretName, keyName string) ([]byte, error) + // + NeedResync() bool + GlobalConfig() (cur, new map[string]string) + GetDirtyIngresses() (del, upd, add []*extensions.Ingress) + GetDirtyEndpoints() []*api.Endpoints + GetDirtyServices() (del, upd, add []*api.Service) + GetDirtySecrets() (del, upd, add []*api.Secret) + GetDirtyPods() []*api.Pod + SyncNewObjects() +} + +// Tracker ... +type Tracker interface { + TrackHostname(rtype ResourceType, name, hostname string) + TrackBackend(rtype ResourceType, name string, backendID hatypes.BackendID) + TrackMissingOnHostname(rtype ResourceType, name, hostname string) + GetDirtyLinks(oldIngressList, oldServiceList, addServiceList, oldSecretList, addSecretList []string) (dirtyIngs, dirtyHosts []string, dirtyBacks []hatypes.BackendID) + DeleteHostnames(hostnames []string) + DeleteBackends(backends []hatypes.BackendID) } // File ... @@ -47,3 +71,17 @@ type CrtFile struct { CommonName string NotAfter time.Time } + +// ResourceType ... +type ResourceType int + +const ( + // IngressType ... + IngressType ResourceType = iota + + // ServiceType ... + ServiceType + + // SecretType ... + SecretType +) diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index 35956705e..2c444d58f 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -40,6 +40,7 @@ type Config interface { Hosts() *hatypes.Hosts Backends() *hatypes.Backends Userlists() []*hatypes.Userlist + Clear() Equals(other Config) bool } @@ -373,6 +374,10 @@ func (c *config) Userlists() []*hatypes.Userlist { return c.userlists } +func (c *config) Clear() { + // IMPLEMENT +} + func (c *config) Equals(other Config) bool { c2, ok := other.(*config) if !ok { diff --git a/pkg/haproxy/types/backend.go b/pkg/haproxy/types/backend.go index 973178306..077fb07ca 100644 --- a/pkg/haproxy/types/backend.go +++ b/pkg/haproxy/types/backend.go @@ -29,6 +29,16 @@ func NewBackendPaths(paths ...*BackendPath) BackendPaths { return b } +// BackendID ... +func (b *Backend) BackendID() BackendID { + // TODO change to an attribute + return BackendID{ + Namespace: b.Namespace, + Name: b.Name, + Port: b.Port, + } +} + // FindEndpoint ... func (b *Backend) FindEndpoint(target string) *Endpoint { for _, endpoint := range b.Endpoints { @@ -39,6 +49,11 @@ func (b *Backend) FindEndpoint(target string) *Endpoint { return nil } +// ClearEndpoints ... +func (b *Backend) ClearEndpoints() { + // IMPLEMENT +} + // AcquireEndpoint ... func (b *Backend) AcquireEndpoint(ip string, port int, targetRef string) *Endpoint { endpoint := b.FindEndpoint(fmt.Sprintf("%s:%d", ip, port)) diff --git a/pkg/haproxy/types/backends.go b/pkg/haproxy/types/backends.go index 2a1e0d70a..8d3a524c3 100644 --- a/pkg/haproxy/types/backends.go +++ b/pkg/haproxy/types/backends.go @@ -54,6 +54,12 @@ func (b *Backends) FindBackend(namespace, name, port string) *Backend { return b.itemsmap[buildID(namespace, name, port)] } +// RemoveAll ... +func (b *Backends) RemoveAll([]BackendID) { + // IMPLEMENT + // rastrear e remover entradas em userlist nao usadas +} + // DefaultBackend ... func (b *Backends) DefaultBackend() *Backend { return b.defaultBackend @@ -84,6 +90,13 @@ func (b *Backends) sortBackends() { }) } +func (b *BackendID) String() string { + if b.id == "" { + b.id = b.Namespace + "_" + b.Name + "_" + b.Port + } + return b.id +} + func createBackend(namespace, name, port string) *Backend { return &Backend{ ID: buildID(namespace, name, port), diff --git a/pkg/haproxy/types/host.go b/pkg/haproxy/types/host.go index a892a081f..6e8730ef9 100644 --- a/pkg/haproxy/types/host.go +++ b/pkg/haproxy/types/host.go @@ -58,6 +58,11 @@ func (h *Hosts) FindHost(hostname string) *Host { return h.itemsmap[hostname] } +// RemoveAll ... +func (h *Hosts) RemoveAll(hostnames []string) { + // IMPLEMENT +} + func (h *Hosts) createHost(hostname string) *Host { return &Host{ Hostname: hostname, diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 44e4e7d0b..f10dea71e 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -363,11 +363,20 @@ type Backends struct { defaultBackend *Backend } +// BackendID ... +type BackendID struct { + id string + Namespace string + Name string + Port string +} + // Backend ... type Backend struct { // // core config // + // TODO use BackendID ID string Namespace string Name string