From 0178122a3381fb0047d534c540ea33dad7f4acb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Thu, 11 May 2023 21:59:39 +0200 Subject: [PATCH 1/4] feat(gwapi): setup controllers dynamically when required CRDs get installed --- Makefile | 1 + config/base/kustomization.yaml | 1 + config/rbac/crds/kustomization.yaml | 3 + config/rbac/crds/role.yaml | 13 + config/rbac/crds/role_binding.yaml | 12 + .../single/all-in-one-dbless-enterprise.yaml | 26 ++ .../all-in-one-dbless-k4k8s-enterprise.yaml | 26 ++ .../all-in-one-dbless-konnect-enterprise.yaml | 26 ++ deploy/single/all-in-one-dbless-konnect.yaml | 26 ++ deploy/single/all-in-one-dbless-legacy.yaml | 26 ++ deploy/single/all-in-one-dbless.yaml | 26 ++ .../all-in-one-postgres-enterprise.yaml | 26 ++ deploy/single/all-in-one-postgres.yaml | 26 ++ .../controllers/crds/dynamic_controller.go | 134 +++++++++++ .../controllers/gateway/gateway_controller.go | 8 + .../gateway/httproute_controller.go | 8 + internal/manager/controllerdef.go | 223 +++++++++--------- internal/manager/scheme/scheme.go | 5 + test/e2e/features_test.go | 43 ++-- 19 files changed, 516 insertions(+), 143 deletions(-) create mode 100644 config/rbac/crds/kustomization.yaml create mode 100644 config/rbac/crds/role.yaml create mode 100644 config/rbac/crds/role_binding.yaml create mode 100644 internal/controllers/crds/dynamic_controller.go diff --git a/Makefile b/Makefile index 581afceb9b..f94647a410 100644 --- a/Makefile +++ b/Makefile @@ -204,6 +204,7 @@ manifests.rbac: controller-gen $(CONTROLLER_GEN) rbac:roleName=kong-ingress paths="./internal/controllers/configuration/" $(CONTROLLER_GEN) rbac:roleName=kong-ingress-knative paths="./internal/controllers/knative/" output:rbac:artifacts:config=config/rbac/knative $(CONTROLLER_GEN) rbac:roleName=kong-ingress-gateway paths="./internal/controllers/gateway/" output:rbac:artifacts:config=config/rbac/gateway + $(CONTROLLER_GEN) rbac:roleName=kong-ingress-crds paths="./internal/controllers/crds/" output:rbac:artifacts:config=config/rbac/crds .PHONY: manifests.single manifests.single: kustomize ## Compose single-file deployment manifests from building blocks diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 3c8df10981..594a45ae19 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../rbac - ../rbac/gateway - ../rbac/knative +- ../rbac/crds - ingressclass.yaml - service.yaml - serviceaccount.yaml diff --git a/config/rbac/crds/kustomization.yaml b/config/rbac/crds/kustomization.yaml new file mode 100644 index 0000000000..b228b96dae --- /dev/null +++ b/config/rbac/crds/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- role.yaml +- role_binding.yaml diff --git a/config/rbac/crds/role.yaml b/config/rbac/crds/role.yaml new file mode 100644 index 0000000000..0ef77edd2c --- /dev/null +++ b/config/rbac/crds/role.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch diff --git a/config/rbac/crds/role_binding.yaml b/config/rbac/crds/role_binding.yaml new file mode 100644 index 0000000000..d8c850b49f --- /dev/null +++ b/config/rbac/crds/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong diff --git a/deploy/single/all-in-one-dbless-enterprise.yaml b/deploy/single/all-in-one-dbless-enterprise.yaml index 4b19168672..036d2865f3 100644 --- a/deploy/single/all-in-one-dbless-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml index 26ff98f993..f41f8f7e54 100644 --- a/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-k4k8s-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml index 3263bfe42e..4ef5697275 100644 --- a/deploy/single/all-in-one-dbless-konnect-enterprise.yaml +++ b/deploy/single/all-in-one-dbless-konnect-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-konnect.yaml b/deploy/single/all-in-one-dbless-konnect.yaml index 5ec982c9dd..61a7adb9aa 100644 --- a/deploy/single/all-in-one-dbless-konnect.yaml +++ b/deploy/single/all-in-one-dbless-konnect.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless-legacy.yaml b/deploy/single/all-in-one-dbless-legacy.yaml index 95c1993c7d..3e33417d99 100644 --- a/deploy/single/all-in-one-dbless-legacy.yaml +++ b/deploy/single/all-in-one-dbless-legacy.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-dbless.yaml b/deploy/single/all-in-one-dbless.yaml index 02ffb9b505..2ed3166055 100644 --- a/deploy/single/all-in-one-dbless.yaml +++ b/deploy/single/all-in-one-dbless.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-postgres-enterprise.yaml b/deploy/single/all-in-one-postgres-enterprise.yaml index bad34705bd..9ac0355bc7 100644 --- a/deploy/single/all-in-one-postgres-enterprise.yaml +++ b/deploy/single/all-in-one-postgres-enterprise.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/deploy/single/all-in-one-postgres.yaml b/deploy/single/all-in-one-postgres.yaml index af5cc18ca5..72f140458c 100644 --- a/deploy/single/all-in-one-postgres.yaml +++ b/deploy/single/all-in-one-postgres.yaml @@ -1374,6 +1374,19 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: kong-ingress-crds +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: kong-ingress-gateway rules: @@ -1550,6 +1563,19 @@ subjects: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: kong-ingress-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-crds +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: kong-ingress-gateway roleRef: diff --git a/internal/controllers/crds/dynamic_controller.go b/internal/controllers/crds/dynamic_controller.go new file mode 100644 index 0000000000..368cf4bf91 --- /dev/null +++ b/internal/controllers/crds/dynamic_controller.go @@ -0,0 +1,134 @@ +package crds + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/samber/lo" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" + "github.com/kong/kubernetes-ingress-controller/v2/internal/util" +) + +// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=list;watch + +type Controller interface { + SetupWithManager(mgr ctrl.Manager) error +} + +// DynamicController ensures that RequiredCRDs are installed in the cluster and only then sets up all of its Controllers +// that depends on them. +// In case the CRDs are not installed at start-up time, DynamicController will set up a watch for CustomResourceDefinition +// and will dynamically set up its Controllers once it detects that all RequiredCRDs are already in place. +type DynamicController struct { + Log logr.Logger + Manager ctrl.Manager + CacheSyncTimeout time.Duration + Controllers []Controller + RequiredCRDs []schema.GroupVersionResource + + startControllersOnce sync.Once +} + +func (r *DynamicController) SetupWithManager(mgr ctrl.Manager) error { + if r.allRequiredCRDsInstalled() { + r.Log.V(util.DebugLevel).Info("All required CustomResourceDefinitions are installed, skipping DynamicController set up") + return r.setupControllers(mgr) + } + + r.Log.Info("Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward") + + c, err := controller.New("DynamicController", mgr, controller.Options{ + Reconciler: r, + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return r.Log + }, + CacheSyncTimeout: r.CacheSyncTimeout, + }) + if err != nil { + return err + } + + return c.Watch( + &source.Kind{Type: &apiextensionsv1.CustomResourceDefinition{}}, + &handler.EnqueueRequestForObject{}, + predicate.NewPredicateFuncs(r.isOneOfRequiredCRDs), + ) +} + +func (r *DynamicController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("CustomResourceDefinition", req.NamespacedName) + + crd := new(apiextensionsv1.CustomResourceDefinition) + if err := r.Manager.GetClient().Get(ctx, req.NamespacedName, crd); err != nil { + if apierrors.IsNotFound(err) { + log.V(util.DebugLevel).Info("Object enqueued no longer exists, skipping", "name", req.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + log.V(util.DebugLevel).Info("Processing CustomResourceDefinition", "name", req.Name) + + if !r.allRequiredCRDsInstalled() { + log.V(util.DebugLevel).Info("Still not all required CustomResourceDefinitions are installed, waiting") + return ctrl.Result{}, nil + } + + var startControllersErr error + r.startControllersOnce.Do(func() { + log.V(util.InfoLevel).Info("All required CustomResourceDefinitions are installed, setting up the controllers") + startControllersErr = r.setupControllers(r.Manager) + }) + if startControllersErr != nil { + return ctrl.Result{}, startControllersErr + } + + return ctrl.Result{}, nil +} + +func (r *DynamicController) allRequiredCRDsInstalled() bool { + return lo.EveryBy(r.RequiredCRDs, func(gvr schema.GroupVersionResource) bool { + return utils.CRDExists(r.Manager.GetClient().RESTMapper(), gvr) + }) +} + +func (r *DynamicController) isOneOfRequiredCRDs(obj client.Object) bool { + crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition) + if !ok { + return false + } + + return lo.ContainsBy(r.RequiredCRDs, func(gvr schema.GroupVersionResource) bool { + versionMatches := lo.ContainsBy(crd.Spec.Versions, func(crdv apiextensionsv1.CustomResourceDefinitionVersion) bool { + return crdv.Name == gvr.Version + }) + + return crd.Spec.Group == gvr.Group && + crd.Status.AcceptedNames.Plural == gvr.Resource && + versionMatches + }) +} + +func (r *DynamicController) setupControllers(mgr ctrl.Manager) error { + errs := lo.FilterMap(r.Controllers, func(c Controller, _ int) (error, bool) { + if err := c.SetupWithManager(mgr); err != nil { + return err, true + } + return nil, false + }) + + return errors.Join(errs...) +} diff --git a/internal/controllers/gateway/gateway_controller.go b/internal/controllers/gateway/gateway_controller.go index 1fda6e94ef..03b8532584 100644 --- a/internal/controllers/gateway/gateway_controller.go +++ b/internal/controllers/gateway/gateway_controller.go @@ -15,6 +15,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,6 +29,7 @@ import ( "github.com/kong/kubernetes-ingress-controller/v2/internal/annotations" ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference" + ctrlutils "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" ) @@ -72,6 +74,12 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("publish service must be configured") } + r.EnableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }) + // generate the controller object and attach it to the manager and link the reconciler object c, err := controller.New("gateway-controller", mgr, controller.Options{ Reconciler: r, diff --git a/internal/controllers/gateway/httproute_controller.go b/internal/controllers/gateway/httproute_controller.go index f594878be0..a1c9367f35 100644 --- a/internal/controllers/gateway/httproute_controller.go +++ b/internal/controllers/gateway/httproute_controller.go @@ -12,6 +12,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + ctrlutils "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/utils" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object" ) @@ -58,6 +60,12 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + r.EnableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }) + // if a GatewayClass updates then we need to enqueue the linked HTTPRoutes to // ensure that any route objects that may have been orphaned by that change get // removed from data-plane configurations, and any routes that are now supported diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 4549348f0e..f7c0bfcc11 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -13,6 +13,7 @@ import ( gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/configuration" + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/crds" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway" "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/knative" ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference" @@ -75,15 +76,6 @@ func setupControllers( return nil, fmt.Errorf("ingress version picker failed: %w", err) } - referenceGrantsEnabled := featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "referencegrants", - }, - restMapper, - ) - referenceIndexers := ctrlref.NewCacheIndexers() controllers := []ControllerDef{ @@ -331,124 +323,123 @@ func setupControllers( // Gateway API Controllers - Beta APIs // --------------------------------------------------------------------------- { - Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gateways", + Enabled: featureGates[featuregates.GatewayFeature], + Controller: &crds.DynamicController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1beta1"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: []schema.GroupVersionResource{ + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gateways", + }, + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gatewayclasses", + }, + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "httproutes", + }, }, - restMapper, - ), - Controller: &gateway.GatewayReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName(featuregates.GatewayFeature), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - PublishServiceRef: c.PublishService.OrEmpty(), - PublishServiceUDPRef: c.PublishServiceUDP, - WatchNamespaces: c.WatchNamespaces, - EnableReferenceGrant: referenceGrantsEnabled, - CacheSyncTimeout: c.CacheSyncTimeout, - ReferenceIndexers: referenceIndexers, - }, - }, - { - Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "httproutes", + Controllers: []crds.Controller{ + &gateway.GatewayReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Gateway"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + PublishServiceRef: c.PublishService.OrEmpty(), + PublishServiceUDPRef: c.PublishServiceUDP, + WatchNamespaces: c.WatchNamespaces, + CacheSyncTimeout: c.CacheSyncTimeout, + ReferenceIndexers: referenceIndexers, + }, + &gateway.HTTPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, }, - restMapper, - ), - Controller: &gateway.HTTPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - EnableReferenceGrant: referenceGrantsEnabled, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, // --------------------------------------------------------------------------- // Gateway API Controllers - Alpha APIs // --------------------------------------------------------------------------- { - Enabled: referenceGrantsEnabled, - Controller: &gateway.ReferenceGrantReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1alpha2"), CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "udproutes", + RequiredCRDs: []schema.GroupVersionResource{ + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "udproutes", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "tcproutes", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "tlsroutes", + }, + { + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "grpcroutes", + }, }, - restMapper, - ), - Controller: &gateway.UDPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "tcproutes", + Controllers: []crds.Controller{ + &gateway.ReferenceGrantReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.UDPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.TCPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.TLSRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + &gateway.GRPCRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, }, - restMapper, - ), - Controller: &gateway.TCPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "tlsroutes", - }, - restMapper, - ), - Controller: &gateway.TLSRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - }, - { - Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController( - schema.GroupVersionResource{ - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "grpcroutes", - }, - restMapper, - ), - Controller: &gateway.GRPCRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, }, }, } diff --git a/internal/manager/scheme/scheme.go b/internal/manager/scheme/scheme.go index 8e3ba4901e..e1318326ed 100644 --- a/internal/manager/scheme/scheme.go +++ b/internal/manager/scheme/scheme.go @@ -1,6 +1,7 @@ package scheme import ( + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" knativev1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1" @@ -18,6 +19,10 @@ import ( func Get(fg map[string]bool) (*runtime.Scheme, error) { scheme := runtime.NewScheme() + if err := apiextensionsv1.AddToScheme(scheme); err != nil { + return nil, err + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { return nil, err } diff --git a/test/e2e/features_test.go b/test/e2e/features_test.go index fa0c89fdf0..d010d53a9f 100644 --- a/test/e2e/features_test.go +++ b/test/e2e/features_test.go @@ -305,23 +305,31 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { LabelSelector: "app=" + controllerDeploymentNN.Name, } - t.Log("verifying that KIC disabled controllers for Gateway API and printed proper log") + t.Log("updating controller deployment to enable alpha Gateway feature gate") + controllerDeployment := deployments.GetController(ctx, t, env) + for i, container := range controllerDeployment.Spec.Template.Spec.Containers { + if container.Name == controllerContainerName { + controllerDeployment.Spec.Template.Spec.Containers[i].Env = append(controllerDeployment.Spec.Template.Spec.Containers[i].Env, + corev1.EnvVar{Name: "CONTROLLER_FEATURE_GATES", Value: consts.DefaultFeatureGates}) + } + } + + _, err := env.Cluster().Client().AppsV1().Deployments(namespace).Update(ctx, controllerDeployment, metav1.UpdateOptions{}) + require.NoError(t, err) + + t.Log("verifying that KIC waits for Gateway API CRDs and prints proper log") require.Eventually(t, func() bool { pods, err := env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).List(ctx, controllerDeploymentListOptions) - gatewayGVR := schema.GroupVersionResource{ - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gateways", - } - msg := fmt.Sprintf("Disabling controller for Group=%s, Resource=%s due to missing CRD", gatewayGVR.GroupVersion(), gatewayGVR.Resource) require.NoError(t, err) + + expectedMsg := "Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward" for _, pod := range pods.Items { logs, err := getPodLogs(ctx, t, env, pod.Namespace, pod.Name) if err != nil { t.Logf("failed to get logs of pods %s/%s, error %v", pod.Namespace, pod.Name, err) return false } - if !strings.Contains(logs, msg) { + if !strings.Contains(logs, expectedMsg) { return false } } @@ -331,14 +339,6 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { t.Logf("deploying Gateway APIs CRDs in standard channel from %s", consts.GatewayStandardCRDsKustomizeURL) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), consts.GatewayStandardCRDsKustomizeURL)) - t.Logf("deleting KIC pods to restart them after Gateway APIs CRDs installed") - pods, err := env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).List(ctx, controllerDeploymentListOptions) - require.NoError(t, err) - for _, pod := range pods.Items { - err = env.Cluster().Client().CoreV1().Pods(controllerDeploymentNN.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - } - t.Log("verifying controller updates associated Gateway resoures") gw := deployGateway(ctx, t, env) verifyGateway(ctx, t, env, gw) @@ -411,17 +411,6 @@ func TestDeployAllInOneDBLESSGateway(t *testing.T) { t.Logf("deploying Gateway APIs CRDs in experimental channel from %s", consts.GatewayExperimentalCRDsKustomizeURL) require.NoError(t, clusters.KustomizeDeployForCluster(ctx, env.Cluster(), consts.GatewayExperimentalCRDsKustomizeURL)) - t.Log("updating controller deployment to enable Gateway feature gate") - controllerDeployment := deployments.GetController(ctx, t, env) - for i, container := range controllerDeployment.Spec.Template.Spec.Containers { - if container.Name == controllerContainerName { - controllerDeployment.Spec.Template.Spec.Containers[i].Env = append(controllerDeployment.Spec.Template.Spec.Containers[i].Env, - corev1.EnvVar{Name: "CONTROLLER_FEATURE_GATES", Value: consts.DefaultFeatureGates}) - } - } - _, err = env.Cluster().Client().AppsV1().Deployments(namespace).Update(ctx, controllerDeployment, metav1.UpdateOptions{}) - require.NoError(t, err) - t.Log("updating proxy deployment to enable TCP listener") proxyDeployment := deployments.GetProxy(ctx, t, env) for i, container := range proxyDeployment.Spec.Template.Spec.Containers { From 9e6b7756229e2a85adb70e44e52e6b943f4ca7cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Mon, 15 May 2023 16:53:55 +0200 Subject: [PATCH 2/4] address review comments --- .../controllers/crds/dynamic_controller.go | 20 +++++++++---------- internal/manager/controllerdef.go | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/controllers/crds/dynamic_controller.go b/internal/controllers/crds/dynamic_controller.go index 368cf4bf91..0bab71009d 100644 --- a/internal/controllers/crds/dynamic_controller.go +++ b/internal/controllers/crds/dynamic_controller.go @@ -29,11 +29,11 @@ type Controller interface { SetupWithManager(mgr ctrl.Manager) error } -// DynamicController ensures that RequiredCRDs are installed in the cluster and only then sets up all of its Controllers +// DynamicCRDController ensures that RequiredCRDs are installed in the cluster and only then sets up all of its Controllers // that depends on them. -// In case the CRDs are not installed at start-up time, DynamicController will set up a watch for CustomResourceDefinition +// In case the CRDs are not installed at start-up time, DynamicCRDController will set up a watch for CustomResourceDefinition // and will dynamically set up its Controllers once it detects that all RequiredCRDs are already in place. -type DynamicController struct { +type DynamicCRDController struct { Log logr.Logger Manager ctrl.Manager CacheSyncTimeout time.Duration @@ -43,15 +43,15 @@ type DynamicController struct { startControllersOnce sync.Once } -func (r *DynamicController) SetupWithManager(mgr ctrl.Manager) error { +func (r *DynamicCRDController) SetupWithManager(mgr ctrl.Manager) error { if r.allRequiredCRDsInstalled() { - r.Log.V(util.DebugLevel).Info("All required CustomResourceDefinitions are installed, skipping DynamicController set up") + r.Log.V(util.DebugLevel).Info("All required CustomResourceDefinitions are installed, skipping DynamicCRDController set up") return r.setupControllers(mgr) } r.Log.Info("Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward") - c, err := controller.New("DynamicController", mgr, controller.Options{ + c, err := controller.New("DynamicCRDController", mgr, controller.Options{ Reconciler: r, LogConstructor: func(_ *reconcile.Request) logr.Logger { return r.Log @@ -69,7 +69,7 @@ func (r *DynamicController) SetupWithManager(mgr ctrl.Manager) error { ) } -func (r *DynamicController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (r *DynamicCRDController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("CustomResourceDefinition", req.NamespacedName) crd := new(apiextensionsv1.CustomResourceDefinition) @@ -99,13 +99,13 @@ func (r *DynamicController) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } -func (r *DynamicController) allRequiredCRDsInstalled() bool { +func (r *DynamicCRDController) allRequiredCRDsInstalled() bool { return lo.EveryBy(r.RequiredCRDs, func(gvr schema.GroupVersionResource) bool { return utils.CRDExists(r.Manager.GetClient().RESTMapper(), gvr) }) } -func (r *DynamicController) isOneOfRequiredCRDs(obj client.Object) bool { +func (r *DynamicCRDController) isOneOfRequiredCRDs(obj client.Object) bool { crd, ok := obj.(*apiextensionsv1.CustomResourceDefinition) if !ok { return false @@ -122,7 +122,7 @@ func (r *DynamicController) isOneOfRequiredCRDs(obj client.Object) bool { }) } -func (r *DynamicController) setupControllers(mgr ctrl.Manager) error { +func (r *DynamicCRDController) setupControllers(mgr ctrl.Manager) error { errs := lo.FilterMap(r.Controllers, func(c Controller, _ int) (error, bool) { if err := c.SetupWithManager(mgr); err != nil { return err, true diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index f7c0bfcc11..81148d338f 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -324,7 +324,7 @@ func setupControllers( // --------------------------------------------------------------------------- { Enabled: featureGates[featuregates.GatewayFeature], - Controller: &crds.DynamicController{ + Controller: &crds.DynamicCRDController{ Manager: mgr, Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1beta1"), CacheSyncTimeout: c.CacheSyncTimeout, @@ -372,7 +372,7 @@ func setupControllers( // --------------------------------------------------------------------------- { Enabled: featureGates[featuregates.GatewayAlphaFeature], - Controller: &crds.DynamicController{ + Controller: &crds.DynamicCRDController{ Manager: mgr, Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1alpha2"), CacheSyncTimeout: c.CacheSyncTimeout, From 73280638f4c095643cbd95c0b0540acbfa0bd4d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Mon, 15 May 2023 19:24:23 +0200 Subject: [PATCH 3/4] granular enablement of controllers --- .../controllers/crds/dynamic_controller.go | 30 +-- internal/manager/controllerdef.go | 237 ++++++++++-------- 2 files changed, 149 insertions(+), 118 deletions(-) diff --git a/internal/controllers/crds/dynamic_controller.go b/internal/controllers/crds/dynamic_controller.go index 0bab71009d..8cae2a0003 100644 --- a/internal/controllers/crds/dynamic_controller.go +++ b/internal/controllers/crds/dynamic_controller.go @@ -2,7 +2,6 @@ package crds import ( "context" - "errors" "sync" "time" @@ -29,15 +28,15 @@ type Controller interface { SetupWithManager(mgr ctrl.Manager) error } -// DynamicCRDController ensures that RequiredCRDs are installed in the cluster and only then sets up all of its Controllers +// DynamicCRDController ensures that RequiredCRDs are installed in the cluster and only then sets up its Controller // that depends on them. // In case the CRDs are not installed at start-up time, DynamicCRDController will set up a watch for CustomResourceDefinition -// and will dynamically set up its Controllers once it detects that all RequiredCRDs are already in place. +// and will dynamically set up its Controller once it detects that all RequiredCRDs are already in place. type DynamicCRDController struct { Log logr.Logger Manager ctrl.Manager CacheSyncTimeout time.Duration - Controllers []Controller + Controller Controller RequiredCRDs []schema.GroupVersionResource startControllersOnce sync.Once @@ -46,7 +45,7 @@ type DynamicCRDController struct { func (r *DynamicCRDController) SetupWithManager(mgr ctrl.Manager) error { if r.allRequiredCRDsInstalled() { r.Log.V(util.DebugLevel).Info("All required CustomResourceDefinitions are installed, skipping DynamicCRDController set up") - return r.setupControllers(mgr) + return r.setupController(mgr) } r.Log.Info("Required CustomResourceDefinitions are not installed, setting up a watch for them in case they are installed afterward") @@ -87,13 +86,13 @@ func (r *DynamicCRDController) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - var startControllersErr error + var startControllerErr error r.startControllersOnce.Do(func() { - log.V(util.InfoLevel).Info("All required CustomResourceDefinitions are installed, setting up the controllers") - startControllersErr = r.setupControllers(r.Manager) + log.V(util.InfoLevel).Info("All required CustomResourceDefinitions are installed, setting up the controller") + startControllerErr = r.setupController(r.Manager) }) - if startControllersErr != nil { - return ctrl.Result{}, startControllersErr + if startControllerErr != nil { + return ctrl.Result{}, startControllerErr } return ctrl.Result{}, nil @@ -122,13 +121,6 @@ func (r *DynamicCRDController) isOneOfRequiredCRDs(obj client.Object) bool { }) } -func (r *DynamicCRDController) setupControllers(mgr ctrl.Manager) error { - errs := lo.FilterMap(r.Controllers, func(c Controller, _ int) (error, bool) { - if err := c.SetupWithManager(mgr); err != nil { - return err, true - } - return nil, false - }) - - return errors.Join(errs...) +func (r *DynamicCRDController) setupController(mgr ctrl.Manager) error { + return r.Controller.SetupWithManager(mgr) } diff --git a/internal/manager/controllerdef.go b/internal/manager/controllerdef.go index 81148d338f..12c7362bae 100644 --- a/internal/manager/controllerdef.go +++ b/internal/manager/controllerdef.go @@ -326,44 +326,39 @@ func setupControllers( Enabled: featureGates[featuregates.GatewayFeature], Controller: &crds.DynamicCRDController{ Manager: mgr, - Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1beta1"), + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/Gateway"), CacheSyncTimeout: c.CacheSyncTimeout, - RequiredCRDs: []schema.GroupVersionResource{ - { - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gateways", - }, - { - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "gatewayclasses", - }, - { - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "httproutes", - }, + RequiredCRDs: baseGatewayCRDs(), + Controller: &gateway.GatewayReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Gateway"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + PublishServiceRef: c.PublishService.OrEmpty(), + PublishServiceUDPRef: c.PublishServiceUDP, + WatchNamespaces: c.WatchNamespaces, + CacheSyncTimeout: c.CacheSyncTimeout, + ReferenceIndexers: referenceIndexers, }, - Controllers: []crds.Controller{ - &gateway.GatewayReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Gateway"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - PublishServiceRef: c.PublishService.OrEmpty(), - PublishServiceUDPRef: c.PublishServiceUDP, - WatchNamespaces: c.WatchNamespaces, - CacheSyncTimeout: c.CacheSyncTimeout, - ReferenceIndexers: referenceIndexers, - }, - &gateway.HTTPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, + }, + }, + { + Enabled: featureGates[featuregates.GatewayFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/HTTPRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "httproutes", + }), + Controller: &gateway.HTTPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("HTTPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, }, }, @@ -374,71 +369,99 @@ func setupControllers( Enabled: featureGates[featuregates.GatewayAlphaFeature], Controller: &crds.DynamicCRDController{ Manager: mgr, - Log: ctrl.Log.WithName("controllers").WithName("Dynamic/gatewayv1alpha2"), + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/ReferenceGrant"), CacheSyncTimeout: c.CacheSyncTimeout, - RequiredCRDs: []schema.GroupVersionResource{ - { - Group: gatewayv1beta1.GroupVersion.Group, - Version: gatewayv1beta1.GroupVersion.Version, - Resource: "referencegrants", - }, - { - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "udproutes", - }, - { - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "tcproutes", - }, - { - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "tlsroutes", - }, - { - Group: gatewayv1alpha2.GroupVersion.Group, - Version: gatewayv1alpha2.GroupVersion.Version, - Resource: "grpcroutes", - }, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "referencegrants", + }), + Controller: &gateway.ReferenceGrantReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, - Controllers: []crds.Controller{ - &gateway.ReferenceGrantReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("ReferenceGrant"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - &gateway.UDPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - &gateway.TCPRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - &gateway.TLSRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, - &gateway.GRPCRouteReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), - Scheme: mgr.GetScheme(), - DataplaneClient: dataplaneClient, - CacheSyncTimeout: c.CacheSyncTimeout, - }, + }, + }, + { + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/UDPRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "udproutes", + }), + Controller: &gateway.UDPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("UDPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + }, + }, + { + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/TCPRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "tcproutes", + }), + Controller: &gateway.TCPRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TCPRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + }, + }, + { + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/TLSRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "tlsroutes", + }), + Controller: &gateway.TLSRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("TLSRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, + }, + }, + }, + { + Enabled: featureGates[featuregates.GatewayAlphaFeature], + Controller: &crds.DynamicCRDController{ + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("Dynamic/GRPCRoute"), + CacheSyncTimeout: c.CacheSyncTimeout, + RequiredCRDs: append(baseGatewayCRDs(), schema.GroupVersionResource{ + Group: gatewayv1alpha2.GroupVersion.Group, + Version: gatewayv1alpha2.GroupVersion.Version, + Resource: "grpcroutes", + }), + Controller: &gateway.GRPCRouteReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("GRPCRoute"), + Scheme: mgr.GetScheme(), + DataplaneClient: dataplaneClient, + CacheSyncTimeout: c.CacheSyncTimeout, }, }, }, @@ -446,3 +469,19 @@ func setupControllers( return controllers, nil } + +// baseGatewayCRDs returns a slice of base CRDs required for running all the Gateway API controllers. +func baseGatewayCRDs() []schema.GroupVersionResource { + return []schema.GroupVersionResource{ + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gateways", + }, + { + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Resource: "gatewayclasses", + }, + } +} From 2978bc1b55f3666819de657a5004f83fb4452cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Tue, 16 May 2023 12:53:33 +0200 Subject: [PATCH 4/4] changelog entry, comment on enableReferenceGrant --- CHANGELOG.md | 4 + .../controllers/gateway/gateway_controller.go | 23 +- .../gateway/httproute_controller.go | 29 +- .../httproute_controller_envtest_test.go | 438 ++++++++---------- 4 files changed, 233 insertions(+), 261 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f196cdd47..85643ca2aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,10 @@ Adding a new version? You'll need three changes: strategy that prevents KIC from exceeding API calls limits. [#3989](https://github.com/Kong/kubernetes-ingress-controller/pull/3989) [#4015](https://github.com/Kong/kubernetes-ingress-controller/pull/4015) +- When Gateway API CRDs are not installed, the controllers of those are not started + during the start-up phase. From now on, they will be dynamically started in runtime + once their installation is detected, making restarting the process unnecessary. + [#3996](https://github.com/Kong/kubernetes-ingress-controller/pull/3996) ### Fixed diff --git a/internal/controllers/gateway/gateway_controller.go b/internal/controllers/gateway/gateway_controller.go index 03b8532584..fef8942fda 100644 --- a/internal/controllers/gateway/gateway_controller.go +++ b/internal/controllers/gateway/gateway_controller.go @@ -55,16 +55,18 @@ type GatewayReconciler struct { //nolint:revive Scheme *runtime.Scheme DataplaneClient *dataplane.KongClient - WatchNamespaces []string - // If EnableReferenceGrant is true, controller will watch ReferenceGrants - // to invalidate or allow cross-namespace TLSConfigs in gateways. - EnableReferenceGrant bool - CacheSyncTimeout time.Duration + WatchNamespaces []string + CacheSyncTimeout time.Duration ReferenceIndexers ctrlref.CacheIndexers PublishServiceRef types.NamespacedName PublishServiceUDPRef mo.Option[types.NamespacedName] + + // If enableReferenceGrant is true, controller will watch ReferenceGrants + // to invalidate or allow cross-namespace TLSConfigs in gateways. + // It's resolved on SetupWithManager call. + enableReferenceGrant bool } // SetupWithManager sets up the controller with the Manager. @@ -74,7 +76,12 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { return fmt.Errorf("publish service must be configured") } - r.EnableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + // We're verifying whether ReferenceGrant CRD is installed at setup of the GatewayReconciler + // to decide whether we should run additional ReferenceGrant watch and handle ReferenceGrants + // when reconciling Gateways. + // Once the GatewayReconciler is set up without ReferenceGrant, there's no possibility to enable + // ReferenceGrant handling again in this reconciler at runtime. + r.enableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ Group: gatewayv1beta1.GroupVersion.Group, Version: gatewayv1beta1.GroupVersion.Version, Resource: "referencegrants", @@ -124,7 +131,7 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { } // watch ReferenceGrants, which may invalidate or allow cross-namespace TLSConfigs - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := c.Watch( &source.Kind{Type: &gatewayv1beta1.ReferenceGrant{}}, handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForGateway), @@ -498,7 +505,7 @@ func (r *GatewayReconciler) reconcileUnmanagedGateway(ctx context.Context, log l // the ReferenceGrants need to be retrieved to ensure that all gateway listeners reference // TLS secrets they are granted for referenceGrantList := &gatewayv1beta1.ReferenceGrantList{} - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := r.Client.List(ctx, referenceGrantList); err != nil { return ctrl.Result{}, err } diff --git a/internal/controllers/gateway/httproute_controller.go b/internal/controllers/gateway/httproute_controller.go index a1c9367f35..b8b2315c3b 100644 --- a/internal/controllers/gateway/httproute_controller.go +++ b/internal/controllers/gateway/httproute_controller.go @@ -37,14 +37,16 @@ import ( type HTTPRouteReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - DataplaneClient DataPlane - // If EnableReferenceGrant is true, we will check for ReferenceGrant if backend in another + Log logr.Logger + Scheme *runtime.Scheme + DataplaneClient DataPlane + CacheSyncTimeout time.Duration + + // If enableReferenceGrant is true, we will check for ReferenceGrant if backend in another // namespace is in backendRefs. // If it is false, referencing backend in different namespace will be rejected. - EnableReferenceGrant bool - CacheSyncTimeout time.Duration + // It's resolved on SetupWithManager call. + enableReferenceGrant bool } // SetupWithManager sets up the controller with the Manager. @@ -60,7 +62,12 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } - r.EnableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ + // We're verifying whether ReferenceGrant CRD is installed at setup of the HTTPRouteReconciler + // to decide whether we should run additional ReferenceGrant watch and handle ReferenceGrants + // when reconciling HTTPRoutes. + // Once the HTTPRouteReconciler is set up without ReferenceGrant, there's no possibility to enable + // ReferenceGrant handling again in this reconciler at runtime. + r.enableReferenceGrant = ctrlutils.CRDExists(mgr.GetRESTMapper(), schema.GroupVersionResource{ Group: gatewayv1beta1.GroupVersion.Group, Version: gatewayv1beta1.GroupVersion.Version, Resource: "referencegrants", @@ -94,7 +101,7 @@ func (r *HTTPRouteReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } - if r.EnableReferenceGrant { + if r.enableReferenceGrant { if err := c.Watch( &source.Kind{Type: &gatewayv1beta1.ReferenceGrant{}}, handler.EnqueueRequestsFromMapFunc(r.listReferenceGrantsForHTTPRoute), @@ -140,8 +147,8 @@ func (r *HTTPRouteReconciler) listReferenceGrantsForHTTPRoute(obj client.Object) for _, gateway := range httproutes.Items { for _, from := range grant.Spec.From { if string(from.Namespace) == gateway.Namespace && - from.Kind == gatewayv1beta1.Kind("HTTPRoute") && - from.Group == gatewayv1beta1.Group("gateway.networking.k8s.io") { + from.Kind == ("HTTPRoute") && + from.Group == ("gateway.networking.k8s.io") { recs = append(recs, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: gateway.Namespace, @@ -673,7 +680,7 @@ func (r *HTTPRouteReconciler) getHTTPRouteRuleReason(ctx context.Context, httpRo // Check if the object referenced is in another namespace, // and if there is grant for that reference if httpRoute.Namespace != backendNamespace { - if !r.EnableReferenceGrant { + if !r.enableReferenceGrant { return gatewayv1beta1.RouteReasonRefNotPermitted, nil } diff --git a/internal/controllers/gateway/httproute_controller_envtest_test.go b/internal/controllers/gateway/httproute_controller_envtest_test.go index fe9d6150b3..c085dff079 100644 --- a/internal/controllers/gateway/httproute_controller_envtest_test.go +++ b/internal/controllers/gateway/httproute_controller_envtest_test.go @@ -52,282 +52,236 @@ func TestHTTPRouteReconcilerProperlyReactsToReferenceGrant(t *testing.T) { require.NoError(t, err) } - // In tests below we use a deferred cancel to stop the manager and not wait - // for its timeout. - - testcases := []struct { - name string - reconciler *gateway.HTTPRouteReconciler - }{ - { - name: "with ReferenceGrant enabled", - reconciler: &gateway.HTTPRouteReconciler{ - Client: client, - EnableReferenceGrant: true, - }, - }, - { - name: "with ReferenceGrant disabled", - reconciler: &gateway.HTTPRouteReconciler{ - Client: client, - EnableReferenceGrant: false, - }, - }, + reconciler := &gateway.HTTPRouteReconciler{ + Client: client, + DataplaneClient: gateway.DataplaneMock{}, } - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + // We use a deferred cancel to stop the manager and not wait for its timeout. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - ns := envtest.CreateNamespace(ctx, t, client) - nsRoute := envtest.CreateNamespace(ctx, t, client) + ns := envtest.CreateNamespace(ctx, t, client) + nsRoute := envtest.CreateNamespace(ctx, t, client) - svc := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: "backend-1", - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - Port: 80, - TargetPort: intstr.FromInt(80), - }, - }, + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: "backend-1", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(80), }, - } - require.NoError(t, client.Create(ctx, &svc)) - - tc.reconciler.DataplaneClient = gateway.DataplaneMock{} - envtest.StartReconciler(ctx, t, client.Scheme(), cfg, tc.reconciler, nil) + }, + }, + } + require.NoError(t, client.Create(ctx, &svc)) + envtest.StartReconciler(ctx, t, client.Scheme(), cfg, reconciler, nil) - gwc := gatewayv1beta1.GatewayClass{ - Spec: gatewayv1beta1.GatewayClassSpec{ - ControllerName: gateway.GetControllerName(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: uuid.NewString(), - Annotations: map[string]string{ - "konghq.com/gatewayclass-unmanaged": "placeholder", - }, - }, - } - require.NoError(t, client.Create(ctx, &gwc)) - t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) + gwc := gatewayv1beta1.GatewayClass{ + Spec: gatewayv1beta1.GatewayClassSpec{ + ControllerName: gateway.GetControllerName(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + Annotations: map[string]string{ + "konghq.com/gatewayclass-unmanaged": "placeholder", + }, + }, + } + require.NoError(t, client.Create(ctx, &gwc)) + t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) - gw := gatewayv1beta1.Gateway{ - Spec: gatewayv1beta1.GatewaySpec{ - GatewayClassName: gatewayv1beta1.ObjectName(gwc.Name), - Listeners: []gatewayv1beta1.Listener{ - { - Name: gatewayv1beta1.SectionName("http"), - Port: gatewayv1beta1.PortNumber(80), - Protocol: gatewayv1beta1.HTTPProtocolType, - AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ - Namespaces: &gatewayv1beta1.RouteNamespaces{ - From: lo.ToPtr(gatewayv1beta1.NamespacesFromAll), - }, - }, + gw := gatewayv1beta1.Gateway{ + Spec: gatewayv1beta1.GatewaySpec{ + GatewayClassName: gatewayv1beta1.ObjectName(gwc.Name), + Listeners: []gatewayv1beta1.Listener{ + { + Name: gatewayv1beta1.SectionName("http"), + Port: gatewayv1beta1.PortNumber(80), + Protocol: gatewayv1beta1.HTTPProtocolType, + AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ + Namespaces: &gatewayv1beta1.RouteNamespaces{ + From: lo.ToPtr(gatewayv1beta1.NamespacesFromAll), }, }, }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: uuid.NewString(), - }, - } - require.NoError(t, client.Create(ctx, &gw)) + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: uuid.NewString(), + }, + } + require.NoError(t, client.Create(ctx, &gw)) - gwOld := gw.DeepCopy() - gw.Status = gatewayv1beta1.GatewayStatus{ - Addresses: []gatewayv1beta1.GatewayAddress{ - { - Type: lo.ToPtr(gatewayv1beta1.IPAddressType), - Value: "10.0.0.1", - }, - }, + gwOld := gw.DeepCopy() + gw.Status = gatewayv1beta1.GatewayStatus{ + Addresses: []gatewayv1beta1.GatewayAddress{ + { + Type: lo.ToPtr(gatewayv1beta1.IPAddressType), + Value: "10.0.0.1", + }, + }, + Conditions: []metav1.Condition{ + { + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "Ready", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + { + Type: "Programmed", + Status: metav1.ConditionTrue, + Reason: "Programmed", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + }, + Listeners: []gatewayv1beta1.ListenerStatus{ + { + Name: "http", Conditions: []metav1.Condition{ - { - Type: "Ready", - Status: metav1.ConditionTrue, - Reason: "Ready", - LastTransitionTime: metav1.Now(), - ObservedGeneration: gw.Generation, - }, { Type: "Accepted", Status: metav1.ConditionTrue, Reason: "Accepted", LastTransitionTime: metav1.Now(), - ObservedGeneration: gw.Generation, }, { - Type: "Programmed", + Type: "Ready", Status: metav1.ConditionTrue, - Reason: "Programmed", + Reason: "Ready", LastTransitionTime: metav1.Now(), - ObservedGeneration: gw.Generation, }, }, - Listeners: []gatewayv1beta1.ListenerStatus{ + SupportedKinds: []gatewayv1beta1.RouteGroupKind{ { - Name: gatewayv1beta1.SectionName("http"), - Conditions: []metav1.Condition{ - { - Type: "Accepted", - Status: metav1.ConditionTrue, - Reason: "Accepted", - LastTransitionTime: metav1.Now(), - }, - { - Type: "Ready", - Status: metav1.ConditionTrue, - Reason: "Ready", - LastTransitionTime: metav1.Now(), - }, - }, - SupportedKinds: []gatewayv1beta1.RouteGroupKind{ - { - Group: lo.ToPtr(gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group)), - Kind: "HTTPRoute", - }, - }, + Group: lo.ToPtr(gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group)), + Kind: "HTTPRoute", }, }, - } - require.NoError(t, client.Status().Patch(ctx, &gw, ctrlclient.MergeFrom(gwOld))) + }, + }, + } + require.NoError(t, client.Status().Patch(ctx, &gw, ctrlclient.MergeFrom(gwOld))) - route := gatewayv1beta1.HTTPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: "HTTPRoute", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: nsRoute.Name, - Name: uuid.NewString(), - }, - Spec: gatewayv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{ - ParentRefs: []gatewayv1beta1.ParentReference{{ - Name: gatewayv1beta1.ObjectName(gw.Name), - Namespace: lo.ToPtr(gatewayv1beta1.Namespace(ns.Name)), - }}, - }, - Rules: []gatewayv1beta1.HTTPRouteRule{{ - BackendRefs: builder.NewHTTPBackendRef("backend-1").WithNamespace(ns.Name).ToSlice(), - }}, - }, - } - require.NoError(t, client.Create(ctx, &route)) + route := gatewayv1beta1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "HTTPRoute", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsRoute.Name, + Name: uuid.NewString(), + }, + Spec: gatewayv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{ + ParentRefs: []gatewayv1beta1.ParentReference{{ + Name: gatewayv1beta1.ObjectName(gw.Name), + Namespace: lo.ToPtr(gatewayv1beta1.Namespace(ns.Name)), + }}, + }, + Rules: []gatewayv1beta1.HTTPRouteRule{{ + BackendRefs: builder.NewHTTPBackendRef("backend-1").WithNamespace(ns.Name).ToSlice(), + }}, + }, + } + require.NoError(t, client.Create(ctx, &route)) - nn := types.NamespacedName{ - Namespace: route.GetNamespace(), - Name: route.GetName(), - } + nn := types.NamespacedName{ + Namespace: route.GetNamespace(), + Name: route.GetName(), + } - t.Logf("verifying that HTTPRoute has ResolvedRefs set to Status False and Reason RefNotPermitted") - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "False", - Reason: "RefNotPermitted", - }, - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } + t.Logf("verifying that HTTPRoute has ResolvedRefs set to Status False and Reason RefNotPermitted") + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "False", + Reason: "RefNotPermitted", + }, + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) + } - rg := gatewayv1beta1.ReferenceGrant{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: uuid.NewString(), + rg := gatewayv1beta1.ReferenceGrant{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: uuid.NewString(), + }, + Spec: gatewayv1beta1.ReferenceGrantSpec{ + From: []gatewayv1beta1.ReferenceGrantFrom{ + { + Group: gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group), + Kind: "HTTPRoute", + Namespace: gatewayv1beta1.Namespace(nsRoute.Name), }, - Spec: gatewayv1beta1.ReferenceGrantSpec{ - From: []gatewayv1beta1.ReferenceGrantFrom{ - { - Group: gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group), - Kind: "HTTPRoute", - Namespace: gatewayv1beta1.Namespace(nsRoute.Name), - }, - }, - To: []gatewayv1beta1.ReferenceGrantTo{ - { - Group: "", - Kind: "Service", - }, - }, + }, + To: []gatewayv1beta1.ReferenceGrantTo{ + { + Group: "", + Kind: "Service", }, - } - require.NoError(t, client.Create(ctx, &rg)) - if tc.reconciler.EnableReferenceGrant { - t.Logf("verifying that HTTPRoute gets accepted by HTTPRouteReconciler after relevant ReferenceGrant gets created") - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "True", - Reason: "ResolvedRefs", - }, - metav1.Condition{ - Type: "Accepted", - Status: "True", - Reason: "Accepted", - }, - // Programmed condition requires a bit more work with mocks. - // It's set only when KubernetesObjectReports are enabled in the underlying - // dataplane client and then it relies on what's returned by - // dataplane client in KubernetesObjectConfigurationStatus(). - // This can be done but it's not the main focus of this test. - // Related: https://github.com/Kong/kubernetes-ingress-controller/issues/3793 - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } - } else { - t.Logf("verifying that HTTPRoute's status doesn't change after relevant ReferenceGrant gets created") - - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyNotContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "True", - Reason: "ResolvedRefs", - }, - metav1.Condition{ - Type: "Accepted", - Status: "True", - Reason: "Accepted", - }, - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } - } + }, + }, + } + require.NoError(t, client.Create(ctx, &rg)) + t.Logf("verifying that HTTPRoute gets accepted by HTTPRouteReconciler after relevant ReferenceGrant gets created") + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "True", + Reason: "ResolvedRefs", + }, + metav1.Condition{ + Type: "Accepted", + Status: "True", + Reason: "Accepted", + }, + // Programmed condition requires a bit more work with mocks. + // It's set only when KubernetesObjectReports are enabled in the underlying + // dataplane client and then it relies on what's returned by + // dataplane client in KubernetesObjectConfigurationStatus(). + // This can be done but it's not the main focus of this test. + // Related: https://github.com/Kong/kubernetes-ingress-controller/issues/3793 + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) + } - require.NoError(t, client.Delete(ctx, &rg)) - t.Logf("verifying that HTTPRoute gets its ResolvedRefs condition to Status False and Reason RefNotPermitted when relevant ReferenceGrant gets deleted") + require.NoError(t, client.Delete(ctx, &rg)) + t.Logf("verifying that HTTPRoute gets its ResolvedRefs condition to Status False and Reason RefNotPermitted when relevant ReferenceGrant gets deleted") - if !assert.Eventually(t, - helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, - metav1.Condition{ - Type: "ResolvedRefs", - Status: "False", - Reason: "RefNotPermitted", - }, - ), - waitDuration, tickDuration, - ) { - t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) - } - }) + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "False", + Reason: "RefNotPermitted", + }, + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) } }