run httproute edge creation

ngrok · Feb 23, 2024 · 97baeb3 · 97baeb3
1 parent e5e6f0b
commit 97baeb3
Show file tree

Hide file tree

Showing 4 changed files with 203 additions and 16 deletions.
diff --git a/internal/controller/gateway/gateway_controller.go b/internal/controller/gateway/gateway_controller.go
@@ -135,8 +135,8 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 // SetupWithManager sets up the controller with the Manager.
 func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	storedResources := []client.Object{
-		//&gatewayv1.GatewayClass{},
-		//&gatewayv1.HTTPRoute{},
+		&gatewayv1.GatewayClass{},
+		&gatewayv1.HTTPRoute{},
 		//&corev1.Service{},
 		&ingressv1alpha1.Domain{},
 		&ingressv1alpha1.HTTPSEdge{},

diff --git a/internal/controller/gateway/httproute_controller.go b/internal/controller/gateway/httproute_controller.go
@@ -27,13 +27,16 @@ package gateway
 import (
 	"context"
 
-	"github.com/go-logr/logr"
-	"github.com/ngrok/kubernetes-ingress-controller/internal/store"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/log"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+
+	"github.com/go-logr/logr"
+	ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/ingress/v1alpha1"
+	"github.com/ngrok/kubernetes-ingress-controller/internal/controller/utils"
+	"github.com/ngrok/kubernetes-ingress-controller/internal/store"
 )
 
 // HTTPRouteReconciler reconciles a HTTPRoute object
@@ -46,21 +49,90 @@ type HTTPRouteReconciler struct {
 	Driver   *store.Driver
 }
 
-//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete
-//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;list;watch;update;patch
-//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/finalizers,verbs=get;list;watch;update
+// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;list;watch;update;patch
 func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	_ = log.FromContext(ctx)
+	log := r.Log.WithValues("HTTPRoute", req.NamespacedName)
+	ctx = ctrl.LoggerInto(ctx, log)
+
+	httproute := new(gatewayv1.HTTPRoute)
+	err := r.Client.Get(ctx, req.NamespacedName, httproute)
+	switch {
+	case err == nil:
+		// all good, continue
+	case client.IgnoreNotFound(err) == nil:
+		if err := r.Driver.DeleteNamedHTTPRoute(req.NamespacedName); err != nil {
+			log.Error(err, "Failed to delete httproute from store")
+			return ctrl.Result{}, err
+		}
 
-	// TODO(user): your logic here
+		err = r.Driver.Sync(ctx, r.Client)
+		if err != nil {
+			log.Error(err, "Failed to sync after removing httproute from store")
+			return ctrl.Result{}, err
+		}
+
+		return ctrl.Result{}, nil
+	default:
+		return ctrl.Result{}, err
+	}
+
+	httproute, err = r.Driver.UpdateHTTPRoute(httproute)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if utils.IsUpsert(httproute) {
+		// The object is not being deleted, so register and sync finalizer
+		if err := utils.RegisterAndSyncFinalizer(ctx, r.Client, httproute); err != nil {
+			log.Error(err, "Failed to register finalizer")
+			return ctrl.Result{}, err
+		}
+	} else {
+		log.Info("Deleting gateway from store")
+		if utils.HasFinalizer(httproute) {
+			if err := utils.RemoveAndSyncFinalizer(ctx, r.Client, httproute); err != nil {
+				log.Error(err, "Failed to remove finalizer")
+				return ctrl.Result{}, err
+			}
+		}
+
+		// Remove it from the store
+		if err := r.Driver.DeleteHTTPRoute(httproute); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
+	if err := r.Driver.Sync(ctx, r.Client); err != nil {
+		log.Error(err, "Faild to sync")
+		return ctrl.Result{}, err
+	}
 
 	return ctrl.Result{}, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	return ctrl.NewControllerManagedBy(mgr).
-		// Uncomment the following line adding a pointer to an instance of the controlled resource as an argument
-		// For().
-		Complete(r)
+	storedResources := []client.Object{
+		&gatewayv1.GatewayClass{},
+		&gatewayv1.Gateway{},
+		//&corev1.Service{},
+		&ingressv1alpha1.Domain{},
+		&ingressv1alpha1.HTTPSEdge{},
+		//&ingressv1alpha1.Tunnel{},
+		//&ingressv1alpha1.NgrokModuleSet{},
+	}
+
+	builder := ctrl.NewControllerManagedBy(mgr).For(&gatewayv1.HTTPRoute{})
+	for _, obj := range storedResources {
+		builder = builder.Watches(
+			obj,
+			store.NewUpdateStoreHandler(
+				obj.GetObjectKind().GroupVersionKind().Kind,
+				r.Driver,
+				r.Client,
+			),
+		)
+	}
+	return builder.Complete(r)
 }
diff --git a/internal/store/cachestores.go b/internal/store/cachestores.go
@@ -36,7 +36,6 @@ type CacheStores struct {
 	ServiceV1      cache.Store
 
 	// Gateway API Stores
-	//HTTPRoute cache.Store
 	Gateway   cache.Store
 	HTTPRoute cache.Store
 

diff --git a/internal/store/driver.go b/internal/store/driver.go
@@ -131,6 +131,7 @@ func (d *Driver) Seed(ctx context.Context, c client.Reader) error {
 
 		httproutes := &gatewayv1.HTTPRouteList{}
 		if err := c.List(ctx, httproutes); err != nil {
+			d.log.Info("WHY IS THIS HAVING AN ISSUE?")
 			return err
 		}
 		for _, httproute := range httproutes.Items {
@@ -256,6 +257,14 @@ func (d *Driver) DeleteNamedGateway(n types.NamespacedName) error {
 	return d.cacheStores.Delete(gtw)
 }
 
+func (d *Driver) DeleteNamedHTTPRoute(n types.NamespacedName) error {
+	httproute := &gatewayv1.HTTPRoute{}
+	// set NamespacedName on the httproute object
+	httproute.SetNamespace(n.Namespace)
+	httproute.SetName(n.Name)
+	return d.cacheStores.Delete(httproute)
+}
+
 // syncStart will:
 //   - let the first caller proceed, indicated by returning true
 //   - while the first one is running any subsequent calls will be batched to the last call
@@ -774,24 +783,98 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph
 
 	for _, gtw := range gateways {
 		for _, listener := range gtw.Spec.Listeners {
+			allowedRoutes := listener.AllowedRoutes.Kinds
+			if len(allowedRoutes) > 0 {
+				createHttpsedge := false
+				for _, routeKind := range allowedRoutes {
+					if routeKind.Kind == "HTTPRoute" {
+						createHttpsedge = true
+					}
+				}
+				if !createHttpsedge {
+					continue
+				}
+			}
 			domainName := string(*listener.Hostname)
 			if _, hasVal := ingressDomains[domainName]; hasVal {
 				// TODO update gateway status if not already updated
 				continue
 			}
+
 			edge, ok := edgeMap[domainName]
 			if !ok {
 				err := errors.NewErrorNotFound(fmt.Sprintf("hostname %v nto found", domainName))
 				d.log.Error(err, "could not find edge associated with rule", "host", domainName)
 				continue
 			}
-
+			// TODO: Set policy from rules.matches and rules.fitlers
 			// skip moduleset and ignore TLS termination for now.
 			if string(*listener.TLS.Mode) != "Terminate" {
 				// set gateway class status here
 				// gtw.Status.Conditions
 				continue
 			}
+			// TODO: Calculate routes from httpRoutes
+			// TODO: skip if no backend services
+			httproutes := d.store.ListHTTPRoutes()
+			for _, httproute := range httproutes {
+				for _, parent := range httproute.Spec.ParentRefs {
+					if string(parent.Name) != gtw.Name {
+						// not our gateway so skip
+						continue
+					}
+					// matches our gateway
+					for _, hostname := range httproute.Spec.Hostnames {
+						if string(hostname) != string(*listener.Hostname) {
+							// doesn't match this listener
+							continue
+						}
+						// matches gateway and listener
+						for _, rule := range httproute.Spec.Rules {
+							// TODO: resolve rule.Matches
+							// TODO: resolve rule.Filters
+							// for v0 we will only resolve the first backendRef
+							for idx, backendref := range rule.BackendRefs {
+								if idx > 0 {
+									break
+								}
+								// handle backendref
+								refKind := string(*backendref.Kind)
+								if refKind != "Serivce" {
+									// only support services currently
+									continue
+								}
+								refName := string(backendref.Name)
+								//refNamespace := string(*backendref.Namespace)
+								serviceUID, servicePort, err := d.getEdgeBackendRef(backendref.BackendRef, gtw.Namespace)
+								if err != nil {
+									d.log.Error(err, "could not find port for service", "namespace", gtw.Namespace, "service", refName)
+								}
+
+								route := ingressv1alpha1.HTTPSEdgeRouteSpec{
+									Match:     "/",           // change based on the rule.match
+									MatchType: "path_prefix", // change based on rule.Matches
+									Backend: ingressv1alpha1.TunnelGroupBackend{
+										Labels: d.ngrokLabels(gtw.Namespace, serviceUID, refName, servicePort),
+									},
+									// TODO: set with values from rules.Filters + rules.Matches
+									//CircuitBreaker:      modSet.Modules.CircuitBreaker,
+									//Compression:         modSet.Modules.Compression,
+									//IPRestriction:       modSet.Modules.IPRestriction,
+									//Headers:             modSet.Modules.Headers,
+									//OAuth:               modSet.Modules.OAuth,
+									//Policy:              modSet.Modules.Policy,
+									//OIDC:                modSet.Modules.OIDC,
+									//SAML:                modSet.Modules.SAML,
+									//WebhookVerification: modSet.Modules.WebhookVerification,
+								}
+								// set different customMetadata for gateways next
+								route.Metadata = d.customMetadata
+							}
+						}
+					}
+				}
+			}
 
 			edgeMap[domainName] = edge
 		}
@@ -911,6 +994,39 @@ func (d *Driver) getEdgeBackend(backendSvc netv1.IngressServiceBackend, namespac
 	return string(service.UID), servicePort.Port, nil
 }
 
+func (d *Driver) getEdgeBackendRef(backendRef gatewayv1.BackendRef, namespace string) (string, int32, error) {
+	service, servicePort, err := d.findBackendRefServicePort(backendRef, namespace)
+	if err != nil {
+		return "", 0, err
+	}
+
+	return string(service.UID), servicePort.Port, nil
+}
+
+func (d *Driver) findBackendRefServicePort(backendRef gatewayv1.BackendRef, namespace string) (*corev1.Service, *corev1.ServicePort, error) {
+	service, err := d.store.GetServiceV1(string(backendRef.Name), namespace)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	servicePort, err := d.findBackendRefServicesPort(service, &backendRef)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return service, servicePort, nil
+}
+
+func (d *Driver) findBackendRefServicesPort(service *corev1.Service, backendRef *gatewayv1.BackendRef) (*corev1.ServicePort, error) {
+	for _, port := range service.Spec.Ports {
+		if (int32(*backendRef.Port) > 0 && port.Port == int32(*backendRef.Port)) || port.Name == string(backendRef.Name) {
+			d.log.V(3).Info("Found matching port for service", "namespace", service.Namespace, "service", service.Name, "port.name", port.Name, "port.number", port.Port)
+			return &port, nil
+		}
+	}
+	return nil, fmt.Errorf("could not find matching port for service %s, backend port %v, name %s", service.Name, int32(*backendRef.Port), string(backendRef.Name))
+}
+
 func (d *Driver) getTunnelBackend(backendSvc netv1.IngressServiceBackend, namespace string) (string, int32, string, string, error) {
 	service, servicePort, err := d.findBackendServicePort(backendSvc, namespace)
 	if err != nil {