From 22a3ead2e9fbf6acd0e512b651ab698b50b404d7 Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Thu, 22 Feb 2024 14:38:49 -0600 Subject: [PATCH 1/8] init httproute controller --- PROJECT | 5 ++ .../templates/rbac/role.yaml | 26 +++++++ .../gateway/httproute_controller.go | 69 +++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 internal/controller/gateway/httproute_controller.go diff --git a/PROJECT b/PROJECT index 337c0a60..78a28d8e 100644 --- a/PROJECT +++ b/PROJECT @@ -75,4 +75,9 @@ resources: group: gateway kind: Gateway version: v1 +- controller: true + domain: k8s.ngrok.com + group: gateway + kind: HTTPRoute + version: v1alpha1 version: "3" diff --git a/helm/ingress-controller/templates/rbac/role.yaml b/helm/ingress-controller/templates/rbac/role.yaml index 3295247e..faa166f9 100644 --- a/helm/ingress-controller/templates/rbac/role.yaml +++ b/helm/ingress-controller/templates/rbac/role.yaml @@ -39,6 +39,32 @@ rules: - get - list - watch +- apiGroups: + - gateway.k8s.ngrok.com + resources: + - httproutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.k8s.ngrok.com + resources: + - httproutes/finalizers + verbs: + - update +- apiGroups: + - gateway.k8s.ngrok.com + resources: + - httproutes/status + verbs: + - get + - patch + - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/internal/controller/gateway/httproute_controller.go b/internal/controller/gateway/httproute_controller.go new file mode 100644 index 00000000..206bab16 --- /dev/null +++ b/internal/controller/gateway/httproute_controller.go @@ -0,0 +1,69 @@ +/* +MIT License + +Copyright (c) 2022 ngrok, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +package gateway + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// HTTPRouteReconciler reconciles a HTTPRoute object +type HTTPRouteReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +//+kubebuilder:rbac:groups=gateway.k8s.ngrok.com,resources=httproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.k8s.ngrok.com,resources=httproutes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=gateway.k8s.ngrok.com,resources=httproutes/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the HTTPRoute object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile +func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + + // TODO(user): your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + // Uncomment the following line adding a pointer to an instance of the controlled resource as an argument + // For(). + Complete(r) +} From 9905cc7416ab6adac7b02962a4c9414ca81a46e4 Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Thu, 22 Feb 2024 14:43:27 -0600 Subject: [PATCH 2/8] only start HTTPRouteReconciler if gateway flag is set --- cmd/main.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index 74236a28..ec928b65 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -287,6 +287,14 @@ func runController(ctx context.Context, opts managerOpts) error { setupLog.Error(err, "unable to create controller", "controller", "Gateway") os.Exit(1) } + + if err = (&gatewaycontroller.HTTPRouteReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute") + os.Exit(1) + } } //+kubebuilder:scaffold:builder From 5f879c1780ae3520b7cbff9bf16dedd48e013339 Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Thu, 22 Feb 2024 14:49:22 -0600 Subject: [PATCH 3/8] setup reconciler struct --- cmd/main.go | 7 ++++-- .../gateway/httproute_controller.go | 22 +++++++------------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index ec928b65..cccbc6a4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -289,8 +289,11 @@ func runController(ctx context.Context, opts managerOpts) error { } if err = (&gatewaycontroller.HTTPRouteReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Gateway"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("gateway-controller"), + Driver: driver, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "HTTPRoute") os.Exit(1) diff --git a/internal/controller/gateway/httproute_controller.go b/internal/controller/gateway/httproute_controller.go index 206bab16..1e978bad 100644 --- a/internal/controller/gateway/httproute_controller.go +++ b/internal/controller/gateway/httproute_controller.go @@ -27,7 +27,10 @@ package gateway import ( "context" + "github.com/go-logr/logr" + "github.com/ngrok/kubernetes-ingress-controller/internal/store" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -36,22 +39,13 @@ import ( // HTTPRouteReconciler reconciles a HTTPRoute object type HTTPRouteReconciler struct { client.Client - Scheme *runtime.Scheme -} -//+kubebuilder:rbac:groups=gateway.k8s.ngrok.com,resources=httproutes,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=gateway.k8s.ngrok.com,resources=httproutes/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=gateway.k8s.ngrok.com,resources=httproutes/finalizers,verbs=update + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + Driver *store.Driver +} -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the HTTPRoute object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) From d601677738ba93987e1e05b7653af9bd6992ac29 Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Thu, 22 Feb 2024 14:50:08 -0600 Subject: [PATCH 4/8] update reconciler markers --- .../templates/rbac/role.yaml | 26 ------------------- .../gateway/httproute_controller.go | 3 +++ 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/helm/ingress-controller/templates/rbac/role.yaml b/helm/ingress-controller/templates/rbac/role.yaml index faa166f9..3295247e 100644 --- a/helm/ingress-controller/templates/rbac/role.yaml +++ b/helm/ingress-controller/templates/rbac/role.yaml @@ -39,32 +39,6 @@ rules: - get - list - watch -- apiGroups: - - gateway.k8s.ngrok.com - resources: - - httproutes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - gateway.k8s.ngrok.com - resources: - - httproutes/finalizers - verbs: - - update -- apiGroups: - - gateway.k8s.ngrok.com - resources: - - httproutes/status - verbs: - - get - - patch - - update - apiGroups: - gateway.networking.k8s.io resources: diff --git a/internal/controller/gateway/httproute_controller.go b/internal/controller/gateway/httproute_controller.go index 1e978bad..4d1799fa 100644 --- a/internal/controller/gateway/httproute_controller.go +++ b/internal/controller/gateway/httproute_controller.go @@ -46,6 +46,9 @@ type HTTPRouteReconciler struct { Driver *store.Driver } +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/finalizers,verbs=get;list;watch;update func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = log.FromContext(ctx) From e37850f477154415cfee439eda10459c98a1191b Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Fri, 23 Feb 2024 12:00:45 -0600 Subject: [PATCH 5/8] run httproute edge creation --- .../controller/gateway/gateway_controller.go | 4 +- .../gateway/httproute_controller.go | 96 +++++++++++++-- internal/store/cachestores.go | 1 - internal/store/driver.go | 116 +++++++++++++++++- 4 files changed, 201 insertions(+), 16 deletions(-) diff --git a/internal/controller/gateway/gateway_controller.go b/internal/controller/gateway/gateway_controller.go index 14bb2424..556fdbde 100644 --- a/internal/controller/gateway/gateway_controller.go +++ b/internal/controller/gateway/gateway_controller.go @@ -135,8 +135,8 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // SetupWithManager sets up the controller with the Manager. func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { storedResources := []client.Object{ - //&gatewayv1.GatewayClass{}, - //&gatewayv1.HTTPRoute{}, + &gatewayv1.GatewayClass{}, + &gatewayv1.HTTPRoute{}, //&corev1.Service{}, &ingressv1alpha1.Domain{}, &ingressv1alpha1.HTTPSEdge{}, diff --git a/internal/controller/gateway/httproute_controller.go b/internal/controller/gateway/httproute_controller.go index 4d1799fa..3d823700 100644 --- a/internal/controller/gateway/httproute_controller.go +++ b/internal/controller/gateway/httproute_controller.go @@ -27,13 +27,16 @@ package gateway import ( "context" - "github.com/go-logr/logr" - "github.com/ngrok/kubernetes-ingress-controller/internal/store" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/go-logr/logr" + ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/ingress/v1alpha1" + "github.com/ngrok/kubernetes-ingress-controller/internal/controller/utils" + "github.com/ngrok/kubernetes-ingress-controller/internal/store" ) // HTTPRouteReconciler reconciles a HTTPRoute object @@ -46,21 +49,90 @@ type HTTPRouteReconciler struct { Driver *store.Driver } -//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;list;watch;update;patch -//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/finalizers,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;list;watch;update;patch func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + log := r.Log.WithValues("HTTPRoute", req.NamespacedName) + ctx = ctrl.LoggerInto(ctx, log) + + httproute := new(gatewayv1.HTTPRoute) + err := r.Client.Get(ctx, req.NamespacedName, httproute) + switch { + case err == nil: + // all good, continue + case client.IgnoreNotFound(err) == nil: + if err := r.Driver.DeleteNamedHTTPRoute(req.NamespacedName); err != nil { + log.Error(err, "Failed to delete httproute from store") + return ctrl.Result{}, err + } - // TODO(user): your logic here + err = r.Driver.Sync(ctx, r.Client) + if err != nil { + log.Error(err, "Failed to sync after removing httproute from store") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + default: + return ctrl.Result{}, err + } + + httproute, err = r.Driver.UpdateHTTPRoute(httproute) + if err != nil { + return ctrl.Result{}, err + } + + if utils.IsUpsert(httproute) { + // The object is not being deleted, so register and sync finalizer + if err := utils.RegisterAndSyncFinalizer(ctx, r.Client, httproute); err != nil { + log.Error(err, "Failed to register finalizer") + return ctrl.Result{}, err + } + } else { + log.Info("Deleting gateway from store") + if utils.HasFinalizer(httproute) { + if err := utils.RemoveAndSyncFinalizer(ctx, r.Client, httproute); err != nil { + log.Error(err, "Failed to remove finalizer") + return ctrl.Result{}, err + } + } + + // Remove it from the store + if err := r.Driver.DeleteHTTPRoute(httproute); err != nil { + return ctrl.Result{}, err + } + } + + if err := r.Driver.Sync(ctx, r.Client); err != nil { + log.Error(err, "Faild to sync") + return ctrl.Result{}, err + } return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - // Uncomment the following line adding a pointer to an instance of the controlled resource as an argument - // For(). - Complete(r) + storedResources := []client.Object{ + &gatewayv1.GatewayClass{}, + &gatewayv1.Gateway{}, + //&corev1.Service{}, + &ingressv1alpha1.Domain{}, + &ingressv1alpha1.HTTPSEdge{}, + //&ingressv1alpha1.Tunnel{}, + //&ingressv1alpha1.NgrokModuleSet{}, + } + + builder := ctrl.NewControllerManagedBy(mgr).For(&gatewayv1.HTTPRoute{}) + for _, obj := range storedResources { + builder = builder.Watches( + obj, + store.NewUpdateStoreHandler( + obj.GetObjectKind().GroupVersionKind().Kind, + r.Driver, + r.Client, + ), + ) + } + return builder.Complete(r) } diff --git a/internal/store/cachestores.go b/internal/store/cachestores.go index cdc00549..7cbf47d9 100644 --- a/internal/store/cachestores.go +++ b/internal/store/cachestores.go @@ -36,7 +36,6 @@ type CacheStores struct { ServiceV1 cache.Store // Gateway API Stores - //HTTPRoute cache.Store Gateway cache.Store HTTPRoute cache.Store diff --git a/internal/store/driver.go b/internal/store/driver.go index 84c06444..b2ff77a0 100644 --- a/internal/store/driver.go +++ b/internal/store/driver.go @@ -256,6 +256,14 @@ func (d *Driver) DeleteNamedGateway(n types.NamespacedName) error { return d.cacheStores.Delete(gtw) } +func (d *Driver) DeleteNamedHTTPRoute(n types.NamespacedName) error { + httproute := &gatewayv1.HTTPRoute{} + // set NamespacedName on the httproute object + httproute.SetNamespace(n.Namespace) + httproute.SetName(n.Name) + return d.cacheStores.Delete(httproute) +} + // syncStart will: // - let the first caller proceed, indicated by returning true // - while the first one is running any subsequent calls will be batched to the last call @@ -782,6 +790,18 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph for _, gtw := range gateways { for _, listener := range gtw.Spec.Listeners { + allowedRoutes := listener.AllowedRoutes.Kinds + if len(allowedRoutes) > 0 { + createHttpsedge := false + for _, routeKind := range allowedRoutes { + if routeKind.Kind == "HTTPRoute" { + createHttpsedge = true + } + } + if !createHttpsedge { + continue + } + } domainName := string(*listener.Hostname) edge, ok := edgeMap[domainName] if !ok { @@ -789,13 +809,74 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph d.log.Error(err, "could not find edge associated with rule", "host", domainName) continue } - + // TODO: Set policy from rules.matches and rules.fitlers // skip moduleset and ignore TLS termination for now. if string(*listener.TLS.Mode) != "Terminate" { // set gateway class status here // gtw.Status.Conditions continue } + // TODO: Calculate routes from httpRoutes + // TODO: skip if no backend services + httproutes := d.store.ListHTTPRoutes() + for _, httproute := range httproutes { + for _, parent := range httproute.Spec.ParentRefs { + if string(parent.Name) != gtw.Name { + // not our gateway so skip + continue + } + // matches our gateway + for _, hostname := range httproute.Spec.Hostnames { + if string(hostname) != string(*listener.Hostname) { + // doesn't match this listener + continue + } + // matches gateway and listener + for _, rule := range httproute.Spec.Rules { + // TODO: resolve rule.Matches + // TODO: resolve rule.Filters + // for v0 we will only resolve the first backendRef + for idx, backendref := range rule.BackendRefs { + if idx > 0 { + break + } + // handle backendref + refKind := string(*backendref.Kind) + if refKind != "Serivce" { + // only support services currently + continue + } + refName := string(backendref.Name) + //refNamespace := string(*backendref.Namespace) + serviceUID, servicePort, err := d.getEdgeBackendRef(backendref.BackendRef, gtw.Namespace) + if err != nil { + d.log.Error(err, "could not find port for service", "namespace", gtw.Namespace, "service", refName) + } + + route := ingressv1alpha1.HTTPSEdgeRouteSpec{ + Match: "/", // change based on the rule.match + MatchType: "path_prefix", // change based on rule.Matches + Backend: ingressv1alpha1.TunnelGroupBackend{ + Labels: d.ngrokLabels(gtw.Namespace, serviceUID, refName, servicePort), + }, + // TODO: set with values from rules.Filters + rules.Matches + //CircuitBreaker: modSet.Modules.CircuitBreaker, + //Compression: modSet.Modules.Compression, + //IPRestriction: modSet.Modules.IPRestriction, + //Headers: modSet.Modules.Headers, + //OAuth: modSet.Modules.OAuth, + //Policy: modSet.Modules.Policy, + //OIDC: modSet.Modules.OIDC, + //SAML: modSet.Modules.SAML, + //WebhookVerification: modSet.Modules.WebhookVerification, + } + // set different customMetadata for gateways next + route.Metadata = d.customMetadata + } + } + } + } + } edgeMap[domainName] = edge } @@ -915,6 +996,39 @@ func (d *Driver) getEdgeBackend(backendSvc netv1.IngressServiceBackend, namespac return string(service.UID), servicePort.Port, nil } +func (d *Driver) getEdgeBackendRef(backendRef gatewayv1.BackendRef, namespace string) (string, int32, error) { + service, servicePort, err := d.findBackendRefServicePort(backendRef, namespace) + if err != nil { + return "", 0, err + } + + return string(service.UID), servicePort.Port, nil +} + +func (d *Driver) findBackendRefServicePort(backendRef gatewayv1.BackendRef, namespace string) (*corev1.Service, *corev1.ServicePort, error) { + service, err := d.store.GetServiceV1(string(backendRef.Name), namespace) + if err != nil { + return nil, nil, err + } + + servicePort, err := d.findBackendRefServicesPort(service, &backendRef) + if err != nil { + return nil, nil, err + } + + return service, servicePort, nil +} + +func (d *Driver) findBackendRefServicesPort(service *corev1.Service, backendRef *gatewayv1.BackendRef) (*corev1.ServicePort, error) { + for _, port := range service.Spec.Ports { + if (int32(*backendRef.Port) > 0 && port.Port == int32(*backendRef.Port)) || port.Name == string(backendRef.Name) { + d.log.V(3).Info("Found matching port for service", "namespace", service.Namespace, "service", service.Name, "port.name", port.Name, "port.number", port.Port) + return &port, nil + } + } + return nil, fmt.Errorf("could not find matching port for service %s, backend port %v, name %s", service.Name, int32(*backendRef.Port), string(backendRef.Name)) +} + func (d *Driver) getTunnelBackend(backendSvc netv1.IngressServiceBackend, namespace string) (string, int32, string, string, error) { service, servicePort, err := d.findBackendServicePort(backendSvc, namespace) if err != nil { From 303b88acd9a6b705222e9339003735f790d52232 Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Mon, 26 Feb 2024 13:30:20 -0600 Subject: [PATCH 6/8] Gateway handles traffic --- Makefile | 4 +- .../templates/rbac/role.yaml | 27 ++++ .../controller/gateway/gateway_controller.go | 16 +-- .../gateway/httproute_controller.go | 11 +- internal/store/driver.go | 130 ++++++++++++++---- internal/store/store.go | 2 +- 6 files changed, 150 insertions(+), 40 deletions(-) diff --git a/Makefile b/Makefile index d5b3f4de..af90f573 100644 --- a/Makefile +++ b/Makefile @@ -57,13 +57,13 @@ preflight: ## Verifies required things like the go version .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=ngrok-ingress-controller-manager-role crd webhook paths="{./api/ingress/v1alpha1/, ./internal/controller/ingress/, ./internal/controller/gateway}" \ + $(CONTROLLER_GEN) rbac:roleName=ngrok-ingress-controller-manager-role crd webhook paths="{./api/ingress/v1alpha1/, ./internal/controller/ingress/, ./internal/controller/gateway/}" \ output:crd:artifacts:config=$(HELM_TEMPLATES_DIR)/crds \ output:rbac:artifacts:config=$(HELM_TEMPLATES_DIR)/rbac .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="{./api/ingress/v1alpha1/, ./internal/controller/ingress/, ./internal/controller/gateway}" + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="{./api/ingress/v1alpha1/, ./internal/controller/ingress/, ./internal/controller/gateway/}" .PHONY: fmt fmt: ## Run go fmt against code. diff --git a/helm/ingress-controller/templates/rbac/role.yaml b/helm/ingress-controller/templates/rbac/role.yaml index 3295247e..55d5c617 100644 --- a/helm/ingress-controller/templates/rbac/role.yaml +++ b/helm/ingress-controller/templates/rbac/role.yaml @@ -23,6 +23,15 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - update + - watch - apiGroups: - "" resources: @@ -75,6 +84,24 @@ rules: - list - update - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes + verbs: + - get + - list + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + verbs: + - get + - list + - update + - watch - apiGroups: - ingress.k8s.ngrok.com resources: diff --git a/internal/controller/gateway/gateway_controller.go b/internal/controller/gateway/gateway_controller.go index 556fdbde..be72e58d 100644 --- a/internal/controller/gateway/gateway_controller.go +++ b/internal/controller/gateway/gateway_controller.go @@ -53,14 +53,14 @@ type GatewayReconciler struct { Driver *store.Driver } -//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch -//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;delete -//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch -//+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch -//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;watch;update -//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/status,verbs=get;list;watch;update -//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;list;watch;update -//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/status,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways/status,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/status,verbs=get;list;watch;update func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("Gateway", req.NamespacedName) diff --git a/internal/controller/gateway/httproute_controller.go b/internal/controller/gateway/httproute_controller.go index 3d823700..e1d08877 100644 --- a/internal/controller/gateway/httproute_controller.go +++ b/internal/controller/gateway/httproute_controller.go @@ -27,6 +27,7 @@ package gateway import ( "context" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" @@ -49,8 +50,10 @@ type HTTPRouteReconciler struct { Driver *store.Driver } -// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;update + func (r *HTTPRouteReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("HTTPRoute", req.NamespacedName) ctx = ctrl.LoggerInto(ctx, log) @@ -116,10 +119,10 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { storedResources := []client.Object{ &gatewayv1.GatewayClass{}, &gatewayv1.Gateway{}, - //&corev1.Service{}, + &corev1.Service{}, &ingressv1alpha1.Domain{}, &ingressv1alpha1.HTTPSEdge{}, - //&ingressv1alpha1.Tunnel{}, + &ingressv1alpha1.Tunnel{}, //&ingressv1alpha1.NgrokModuleSet{}, } diff --git a/internal/store/driver.go b/internal/store/driver.go index b2ff77a0..600c9b15 100644 --- a/internal/store/driver.go +++ b/internal/store/driver.go @@ -129,15 +129,15 @@ func (d *Driver) Seed(ctx context.Context, c client.Reader) error { } } - //httproutes := &gatewayv1.HTTPRouteList{} - //if err := c.List(ctx, httproutes); err != nil { - // return err - //} - //for _, httproute := range httproutes.Items { - // if err := d.store.Update(&httproute); err != nil { - // return err - // } - //} + httproutes := &gatewayv1.HTTPRouteList{} + if err := c.List(ctx, httproutes); err != nil { + return err + } + for _, httproute := range httproutes.Items { + if err := d.store.Update(&httproute); err != nil { + return err + } + } } services := &corev1.ServiceList{} @@ -811,11 +811,11 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph } // TODO: Set policy from rules.matches and rules.fitlers // skip moduleset and ignore TLS termination for now. - if string(*listener.TLS.Mode) != "Terminate" { - // set gateway class status here - // gtw.Status.Conditions - continue - } + //if string(*listener.TLS.Mode) != "Terminate" { + // // set gateway class status here + // // gtw.Status.Conditions + // continue + //} // TODO: Calculate routes from httpRoutes // TODO: skip if no backend services httproutes := d.store.ListHTTPRoutes() @@ -842,7 +842,7 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph } // handle backendref refKind := string(*backendref.Kind) - if refKind != "Serivce" { + if refKind != "Service" { // only support services currently continue } @@ -860,18 +860,12 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph Labels: d.ngrokLabels(gtw.Namespace, serviceUID, refName, servicePort), }, // TODO: set with values from rules.Filters + rules.Matches - //CircuitBreaker: modSet.Modules.CircuitBreaker, - //Compression: modSet.Modules.Compression, - //IPRestriction: modSet.Modules.IPRestriction, - //Headers: modSet.Modules.Headers, - //OAuth: modSet.Modules.OAuth, //Policy: modSet.Modules.Policy, - //OIDC: modSet.Modules.OIDC, - //SAML: modSet.Modules.SAML, - //WebhookVerification: modSet.Modules.WebhookVerification, } // set different customMetadata for gateways next route.Metadata = d.customMetadata + + edge.Spec.Routes = append(edge.Spec.Routes, route) } } } @@ -899,7 +893,12 @@ func (d *Driver) tunnelKeyFromTunnel(tunnel ingressv1alpha1.Tunnel) tunnelKey { func (d *Driver) calculateTunnels() map[tunnelKey]ingressv1alpha1.Tunnel { tunnels := map[tunnelKey]ingressv1alpha1.Tunnel{} + d.calculateTunnelsFromIngress(tunnels) + d.calculateTunnelsFromGateway(tunnels) + return tunnels +} +func (d *Driver) calculateTunnelsFromIngress(tunnels map[tunnelKey]ingressv1alpha1.Tunnel) { for _, ingress := range d.store.ListNgrokIngressesV1() { for _, rule := range ingress.Spec.Rules { for _, path := range rule.HTTP.Paths { @@ -959,8 +958,70 @@ func (d *Driver) calculateTunnels() map[tunnelKey]ingressv1alpha1.Tunnel { } } } +} - return tunnels +func (d *Driver) calculateTunnelsFromGateway(tunnels map[tunnelKey]ingressv1alpha1.Tunnel) { + httproutes := d.store.ListHTTPRoutes() + + for _, httproute := range httproutes { + for _, rule := range httproute.Spec.Rules { + for _, backendRef := range rule.BackendRefs { + // We only support service backends right now. TODO: support resource backends + //if path.Backend.Service == nil { + // continue + //} + + serviceName := string(backendRef.Name) + serviceUID, servicePort, protocol, appProtocol, err := d.getTunnelBackendFromGateway(backendRef.BackendRef, httproute.Namespace) + if err != nil { + d.log.Error(err, "could not find port for service", "namespace", httproute.Namespace, "service", serviceName) + } + + key := tunnelKey{httproute.Namespace, serviceName, strconv.Itoa(int(servicePort))} + tunnel, found := tunnels[key] + if !found { + targetAddr := fmt.Sprintf("%s.%s.%s:%d", serviceName, key.namespace, clusterDomain, servicePort) + tunnel = ingressv1alpha1.Tunnel{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-%d-", serviceName, servicePort), + Namespace: httproute.Namespace, + OwnerReferences: nil, // fill owner references below + Labels: d.tunnelLabels(serviceName, servicePort), + }, + Spec: ingressv1alpha1.TunnelSpec{ + ForwardsTo: targetAddr, + Labels: d.ngrokLabels(httproute.Namespace, serviceUID, serviceName, servicePort), + BackendConfig: &ingressv1alpha1.BackendConfig{ + Protocol: protocol, + }, + AppProtocol: appProtocol, + }, + } + } + + hasIngressReference := false + for _, ref := range tunnel.OwnerReferences { + if ref.UID == httproute.UID { + hasIngressReference = true + break + } + } + if !hasIngressReference { + tunnel.OwnerReferences = append(tunnel.OwnerReferences, metav1.OwnerReference{ + APIVersion: httproute.APIVersion, + Kind: httproute.Kind, + Name: httproute.Name, + UID: httproute.UID, + }) + slices.SortStableFunc(tunnel.OwnerReferences, func(i, j metav1.OwnerReference) int { + return cmp.Compare(string(i.UID), string(j.UID)) + }) + } + + tunnels[key] = tunnel + } + } + } } func (d *Driver) calculateIngressLoadBalancerIPStatus(ing *netv1.Ingress, c client.Reader) []netv1.IngressLoadBalancerIngress { @@ -1010,7 +1071,7 @@ func (d *Driver) findBackendRefServicePort(backendRef gatewayv1.BackendRef, name if err != nil { return nil, nil, err } - + d.log.Info("TESTING", "backendRef.Name", backendRef.Name) servicePort, err := d.findBackendRefServicesPort(service, &backendRef) if err != nil { return nil, nil, err @@ -1048,6 +1109,25 @@ func (d *Driver) getTunnelBackend(backendSvc netv1.IngressServiceBackend, namesp return string(service.UID), servicePort.Port, protocol, appProtocol, nil } +func (d *Driver) getTunnelBackendFromGateway(backendRef gatewayv1.BackendRef, namespace string) (string, int32, string, string, error) { + service, servicePort, err := d.findBackendRefServicePort(backendRef, namespace) + if err != nil { + return "", 0, "", "", err + } + + protocol, err := d.getPortAnnotatedProtocol(service, servicePort.Name) + if err != nil { + return "", 0, "", "", err + } + + appProtocol, err := d.getPortAppProtocol(service, servicePort) + if err != nil { + return "", 0, "", "", err + } + + return string(service.UID), servicePort.Port, protocol, appProtocol, nil +} + func (d *Driver) findBackendServicePort(backendSvc netv1.IngressServiceBackend, namespace string) (*corev1.Service, *corev1.ServicePort, error) { service, err := d.store.GetServiceV1(backendSvc.Name, namespace) if err != nil { diff --git a/internal/store/store.go b/internal/store/store.go index 1f76b8bd..557b9b58 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -176,7 +176,7 @@ func (s Store) GetGateway(name string, namespace string) (*gatewayv1.Gateway, er return gtw.(*gatewayv1.Gateway), nil } -func (s Store) GetHTTPRoute(namespace string, name string) (*gatewayv1.HTTPRoute, error) { +func (s Store) GetHTTPRoute(name string, namespace string) (*gatewayv1.HTTPRoute, error) { obj, exists, err := s.stores.HTTPRoute.GetByKey(getKey(name, namespace)) if err != nil { return nil, err From 65371542d5e898a3f4d532c9ecb2d29bb21b0b8d Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Mon, 4 Mar 2024 08:32:56 -0600 Subject: [PATCH 7/8] remove debug log --- internal/store/driver.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/store/driver.go b/internal/store/driver.go index 600c9b15..7e1b764b 100644 --- a/internal/store/driver.go +++ b/internal/store/driver.go @@ -1071,7 +1071,6 @@ func (d *Driver) findBackendRefServicePort(backendRef gatewayv1.BackendRef, name if err != nil { return nil, nil, err } - d.log.Info("TESTING", "backendRef.Name", backendRef.Name) servicePort, err := d.findBackendRefServicesPort(service, &backendRef) if err != nil { return nil, nil, err From e781f6e38b0d398d381285acc758429711fd52d0 Mon Sep 17 00:00:00 2001 From: Abdirahman Osman Date: Mon, 4 Mar 2024 10:36:39 -0600 Subject: [PATCH 8/8] map HTTProute to HTTPSEdge --- internal/store/driver.go | 113 +++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 23 deletions(-) diff --git a/internal/store/driver.go b/internal/store/driver.go index 7e1b764b..fd3fae0d 100644 --- a/internal/store/driver.go +++ b/internal/store/driver.go @@ -352,9 +352,9 @@ func (d *Driver) Sync(ctx context.Context, c client.Client) error { } d.log.Info("syncing driver state!!") - desiredDomains, desiredIngressDomains, desiredGatewayDomains := d.calculateDomains() + desiredDomains, desiredIngressDomains, desiredGatewayDomainMap := d.calculateDomains() - desiredEdges := d.calculateHTTPSEdges(&desiredIngressDomains, &desiredGatewayDomains) + desiredEdges := d.calculateHTTPSEdges(&desiredIngressDomains, desiredGatewayDomainMap) desiredTunnels := d.calculateTunnels() currDomains := &ingressv1alpha1.DomainList{} @@ -419,9 +419,9 @@ func (d *Driver) SyncEdges(ctx context.Context, c client.Client) error { } d.log.Info("syncing edges state!!") - _, desiredIngressDomains, desiredGatewayDomains := d.calculateDomains() + _, desiredIngressDomains, desiredGatewayDomainMap := d.calculateDomains() - desiredEdges := d.calculateHTTPSEdges(&desiredIngressDomains, &desiredGatewayDomains) + desiredEdges := d.calculateHTTPSEdges(&desiredIngressDomains, desiredGatewayDomainMap) currEdges := &ingressv1alpha1.HTTPSEdgeList{} if err := c.List(ctx, currEdges, client.MatchingLabels{ labelControllerNamespace: d.managerName.Namespace, @@ -583,8 +583,8 @@ func (d *Driver) updateIngressStatuses(ctx context.Context, c client.Client) err // return nil //} -func (d *Driver) calculateDomains() ([]ingressv1alpha1.Domain, []ingressv1alpha1.Domain, []ingressv1alpha1.Domain) { - var domains, ingressDomains, gatewayDomains []ingressv1alpha1.Domain +func (d *Driver) calculateDomains() ([]ingressv1alpha1.Domain, []ingressv1alpha1.Domain, map[string]ingressv1alpha1.Domain) { + var domains, ingressDomains []ingressv1alpha1.Domain ingressDomainMap := d.calculateDomainsFromIngress() ingressDomains = make([]ingressv1alpha1.Domain, 0, len(ingressDomainMap)) @@ -593,17 +593,17 @@ func (d *Driver) calculateDomains() ([]ingressv1alpha1.Domain, []ingressv1alpha1 domains = append(domains, domain) } + var gatewayDomainMap map[string]ingressv1alpha1.Domain if d.gatewayEnabled { gatewayDomainMap := d.calculateDomainsFromGateway(ingressDomainMap) domains := make([]ingressv1alpha1.Domain, 0, len(gatewayDomainMap)) for _, domain := range gatewayDomainMap { domains = append(domains, domain) - gatewayDomains = append(gatewayDomains, domain) } } - return domains, ingressDomains, gatewayDomains + return domains, ingressDomains, gatewayDomainMap } func (d *Driver) calculateDomainsFromIngress() map[string]ingressv1alpha1.Domain { @@ -688,7 +688,7 @@ func (d *Driver) getNgrokModuleSetForIngress(ing *netv1.Ingress) (*ingressv1alph return computedModSet, nil } -func (d *Driver) calculateHTTPSEdges(ingressDomains *[]ingressv1alpha1.Domain, gatewayDomains *[]ingressv1alpha1.Domain) map[string]ingressv1alpha1.HTTPSEdge { +func (d *Driver) calculateHTTPSEdges(ingressDomains *[]ingressv1alpha1.Domain, gatewayDomainMap map[string]ingressv1alpha1.Domain) map[string]ingressv1alpha1.HTTPSEdge { edgeMap := make(map[string]ingressv1alpha1.HTTPSEdge, len(*ingressDomains)) for _, domain := range *ingressDomains { edge := ingressv1alpha1.HTTPSEdge{ @@ -706,6 +706,68 @@ func (d *Driver) calculateHTTPSEdges(ingressDomains *[]ingressv1alpha1.Domain, g } d.calculateHTTPSEdgesFromIngress(edgeMap) + if d.gatewayEnabled { + + httproutes := d.store.ListHTTPRoutes() + gateways := d.store.ListGateways() + for _, gtw := range gateways { + var gatewayDomains []string + for _, listener := range gtw.Spec.Listeners { + if listener.Hostname == nil { + continue + } + gatewayDomains = append(gatewayDomains, string(*listener.Hostname)) + } + if len(gatewayDomains) <= 0 { + continue + } + for _, httproute := range httproutes { + var routeDomains []string + for _, parent := range httproute.Spec.ParentRefs { + if string(parent.Name) != gtw.Name { + continue + } + var domainOverlap []string + for _, hostname := range httproute.Spec.Hostnames { + for _, parentDomain := range gatewayDomains { + if parentDomain == string(hostname) { + domainOverlap = append(domainOverlap, parentDomain) + } + } + } + if len(domainOverlap) <= 0 { + // no hostnames overlap with gateway + continue + } + routeDomains = append(routeDomains, domainOverlap...) + } + if len(routeDomains) <= 0 { + // no usable domains in route + continue + } + var hostPorts []string + + for _, domain := range routeDomains { + hostPorts = append(hostPorts, domain+":443") + } + edge := ingressv1alpha1.HTTPSEdge{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: httproute.Name + "-", + Namespace: httproute.Namespace, + Labels: d.edgeLabels(routeDomains[0]), + }, + Spec: ingressv1alpha1.HTTPSEdgeSpec{ + Hostports: hostPorts, + }, + } + edge.Spec.Metadata = d.customMetadata + edgeMap[routeDomains[0]] = edge + + } + } + d.calculateHTTPSEdgesFromGateway(edgeMap) + } + return edgeMap } @@ -785,7 +847,7 @@ func (d *Driver) calculateHTTPSEdgesFromIngress(edgeMap map[string]ingressv1alph } } -func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alpha1.HTTPSEdge, gatewayDomains *[]ingressv1alpha1.Domain) { +func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alpha1.HTTPSEdge) { gateways := d.store.ListGateways() for _, gtw := range gateways { @@ -805,8 +867,8 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph domainName := string(*listener.Hostname) edge, ok := edgeMap[domainName] if !ok { - err := errors.NewErrorNotFound(fmt.Sprintf("hostname %v nto found", domainName)) - d.log.Error(err, "could not find edge associated with rule", "host", domainName) + //err := errors.NewErrorNotFound(fmt.Sprintf("hostname %v not found", domainName)) + d.log.Info("could not find edge associated with rule", "host", domainName) continue } // TODO: Set policy from rules.matches and rules.fitlers @@ -836,7 +898,17 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph // TODO: resolve rule.Matches // TODO: resolve rule.Filters // for v0 we will only resolve the first backendRef + route := ingressv1alpha1.HTTPSEdgeRouteSpec{ + Match: "/", // change based on the rule.match + MatchType: "path_prefix", // change based on rule.Matches + //}, + // TODO: set with values from rules.Filters + rules.Matches + //Policy: modSet.Modules.Policy, + } + for idx, backendref := range rule.BackendRefs { + // currently the ingress controller doesn't support weighted backends + // so we'll only support one backendref per rule if idx > 0 { break } @@ -853,20 +925,15 @@ func (d *Driver) calculateHTTPSEdgesFromGateway(edgeMap map[string]ingressv1alph d.log.Error(err, "could not find port for service", "namespace", gtw.Namespace, "service", refName) } - route := ingressv1alpha1.HTTPSEdgeRouteSpec{ - Match: "/", // change based on the rule.match - MatchType: "path_prefix", // change based on rule.Matches - Backend: ingressv1alpha1.TunnelGroupBackend{ - Labels: d.ngrokLabels(gtw.Namespace, serviceUID, refName, servicePort), - }, - // TODO: set with values from rules.Filters + rules.Matches - //Policy: modSet.Modules.Policy, + route.Backend = ingressv1alpha1.TunnelGroupBackend{ + Labels: d.ngrokLabels(gtw.Namespace, serviceUID, refName, servicePort), } - // set different customMetadata for gateways next - route.Metadata = d.customMetadata - edge.Spec.Routes = append(edge.Spec.Routes, route) } + // set different customMetadata for gateways next + route.Metadata = d.customMetadata + + edge.Spec.Routes = append(edge.Spec.Routes, route) } } }