Skip to content

Commit

Permalink
feat: make scheme conditional on provided feature gates
Browse files Browse the repository at this point in the history
  • Loading branch information
pmalek committed Feb 13, 2023
1 parent 2ff6462 commit 1218fd1
Show file tree
Hide file tree
Showing 7 changed files with 99 additions and 77 deletions.
3 changes: 2 additions & 1 deletion internal/manager/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/kong/kubernetes-ingress-controller/v2/internal/annotations"
"github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway"
"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/featuregates"
)

// -----------------------------------------------------------------------------
Expand Down Expand Up @@ -224,7 +225,7 @@ func (c *Config) FlagSet() *pflag.FlagSet {

// Feature Gates (see FEATURE_GATES.md)
flagSet.Var(cliflag.NewMapStringBool(&c.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/beta/experimental features. "+
fmt.Sprintf("See the Feature Gates documentation for information and available options: %s", featureGatesDocsURL))
fmt.Sprintf("See the Feature Gates documentation for information and available options: %s", featuregates.DocsURL))

// SIGTERM or SIGINT signal delay
flagSet.DurationVar(&c.TermDelay, "term-delay", time.Second*0, "The time delay to sleep before SIGTERM or SIGINT will shut down the Ingress Controller")
Expand Down
17 changes: 9 additions & 8 deletions internal/manager/controllerdef.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/knative"
ctrlref "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/reference"
"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/featuregates"
"github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object/status"
konghqcomv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
konghqcomv1alpha1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1alpha1"
Expand Down Expand Up @@ -73,7 +74,7 @@ func setupControllers(
return nil, fmt.Errorf("ingress version picker failed: %w", err)
}

referenceGrantsEnabled := featureGates[gatewayFeature] && ShouldEnableCRDController(
referenceGrantsEnabled := featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController(
schema.GroupVersionResource{
Group: gatewayv1beta1.GroupVersion.Group,
Version: gatewayv1beta1.GroupVersion.Version,
Expand Down Expand Up @@ -337,7 +338,7 @@ func setupControllers(
// knative is a special case because it existed before we added feature gates functionality
// for this controller (only) the existing --enable-controller-knativeingress flag overrides
// any feature gate configuration. See FEATURE_GATES.md for more information.
Enabled: (featureGates[knativeFeature] || c.KnativeIngressEnabled) && ShouldEnableCRDController(
Enabled: (featureGates[featuregates.KnativeFeature] || c.KnativeIngressEnabled) && ShouldEnableCRDController(
schema.GroupVersionResource{
Group: knativev1alpha1.SchemeGroupVersion.Group,
Version: knativev1alpha1.SchemeGroupVersion.Version,
Expand All @@ -362,7 +363,7 @@ func setupControllers(
// Gateway API Controllers - Beta APIs
// ---------------------------------------------------------------------------
{
Enabled: featureGates[gatewayFeature] && ShouldEnableCRDController(
Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController(
schema.GroupVersionResource{
Group: gatewayv1beta1.GroupVersion.Group,
Version: gatewayv1beta1.GroupVersion.Version,
Expand All @@ -372,7 +373,7 @@ func setupControllers(
),
Controller: &gateway.GatewayReconciler{
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName(gatewayFeature),
Log: ctrl.Log.WithName("controllers").WithName(featuregates.GatewayFeature),
Scheme: mgr.GetScheme(),
DataplaneClient: dataplaneClient,
PublishService: c.PublishService.String(),
Expand All @@ -383,7 +384,7 @@ func setupControllers(
},
},
{
Enabled: featureGates[gatewayFeature] && ShouldEnableCRDController(
Enabled: featureGates[featuregates.GatewayFeature] && ShouldEnableCRDController(
schema.GroupVersionResource{
Group: gatewayv1beta1.GroupVersion.Group,
Version: gatewayv1beta1.GroupVersion.Version,
Expand Down Expand Up @@ -414,7 +415,7 @@ func setupControllers(
},
},
{
Enabled: featureGates[gatewayAlphaFeature] && ShouldEnableCRDController(
Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController(
schema.GroupVersionResource{
Group: gatewayv1alpha2.GroupVersion.Group,
Version: gatewayv1alpha2.GroupVersion.Version,
Expand All @@ -431,7 +432,7 @@ func setupControllers(
},
},
{
Enabled: featureGates[gatewayAlphaFeature] && ShouldEnableCRDController(
Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController(
schema.GroupVersionResource{
Group: gatewayv1alpha2.GroupVersion.Group,
Version: gatewayv1alpha2.GroupVersion.Version,
Expand All @@ -448,7 +449,7 @@ func setupControllers(
},
},
{
Enabled: featureGates[gatewayAlphaFeature] && ShouldEnableCRDController(
Enabled: featureGates[featuregates.GatewayAlphaFeature] && ShouldEnableCRDController(
schema.GroupVersionResource{
Group: gatewayv1alpha2.GroupVersion.Group,
Version: gatewayv1alpha2.GroupVersion.Version,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package manager
package featuregates

import (
"fmt"
Expand All @@ -11,53 +11,53 @@ import (
// -----------------------------------------------------------------------------

const (
// knativeFeature is the name of the feature-gate for enabling/disabling Knative.
knativeFeature = "Knative"
// KnativeFeature is the name of the feature-gate for enabling/disabling Knative.
KnativeFeature = "Knative"

// gatewayFeature is the name of the feature-gate for enabling/disabling Gateway APIs.
gatewayFeature = "Gateway"
// GatewayFeature is the name of the feature-gate for enabling/disabling GatewayFeature APIs.
GatewayFeature = "Gateway"

// gatewayAlphaFeature is the name of the feature-gate for enabling or
// GatewayAlphaFeature is the name of the feature-gate for enabling or
// disabling the Alpha maturity APIs and relevant features for Gateway API.
gatewayAlphaFeature = "GatewayAlpha"
GatewayAlphaFeature = "GatewayAlpha"

// combinedRoutesFeature is the name of the feature-gate for the newer object
// CombinedRoutesFeature is the name of the feature-gate for the newer object
// translation logic that will combine routes for kong services when translating
// objects like Ingress instead of creating a route per path.
combinedRoutesFeature = "CombinedRoutes"
CombinedRoutesFeature = "CombinedRoutes"

// featureGatesDocsURL provides a link to the documentation for feature gates in the KIC repository.
featureGatesDocsURL = "https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md"
// DocsURL provides a link to the documentation for feature gates in the KIC repository.
DocsURL = "https://github.com/Kong/kubernetes-ingress-controller/blob/main/FEATURE_GATES.md"
)

// setupFeatureGates converts feature gates to controller enablement.
func setupFeatureGates(setupLog logr.Logger, featureGates map[string]bool) (map[string]bool, error) {
// Setup converts feature gates to controller enablement.
func Setup(setupLog logr.Logger, featureGates map[string]bool) (map[string]bool, error) {
// generate a map of feature gates by string names to their controller enablement
ctrlMap := getFeatureGatesDefaults()
ctrlMap := GetFeatureGatesDefaults()

// override the default settings
for feature, enabled := range featureGates {
setupLog.Info("found configuration option for gated feature", "feature", feature, "enabled", enabled)
_, ok := ctrlMap[feature]
if !ok {
return ctrlMap, fmt.Errorf("%s is not a valid feature, please see the documentation: %s", feature, featureGatesDocsURL)
return ctrlMap, fmt.Errorf("%s is not a valid feature, please see the documentation: %s", feature, DocsURL)
}
ctrlMap[feature] = enabled
}

return ctrlMap, nil
}

// getFeatureGatesDefaults initializes a feature gate map given the currently
// GetFeatureGatesDefaults initializes a feature gate map given the currently
// supported feature gates options and derives defaults for them based on
// manager configuration options if present.
//
// NOTE: if you're adding a new feature gate, it needs to be added here.
func getFeatureGatesDefaults() map[string]bool {
func GetFeatureGatesDefaults() map[string]bool {
return map[string]bool{
knativeFeature: false,
gatewayFeature: true,
gatewayAlphaFeature: false,
combinedRoutesFeature: true,
KnativeFeature: false,
GatewayFeature: true,
GatewayAlphaFeature: false,
CombinedRoutesFeature: true,
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package manager
package featuregates

import (
"bytes"
Expand All @@ -18,21 +18,21 @@ func TestFeatureGates(t *testing.T) {
setupLog := logrusr.New(baseLogger)

t.Log("verifying feature gates setup defaults when no feature gates are configured")
fgs, err := setupFeatureGates(setupLog, nil)
fgs, err := Setup(setupLog, nil)
assert.NoError(t, err)
assert.Len(t, fgs, len(getFeatureGatesDefaults()))
assert.Len(t, fgs, len(GetFeatureGatesDefaults()))

t.Log("verifying feature gates setup results when valid feature gates options are present")
featureGates := map[string]bool{gatewayFeature: true}
fgs, err = setupFeatureGates(setupLog, featureGates)
featureGates := map[string]bool{GatewayFeature: true}
fgs, err = Setup(setupLog, featureGates)
assert.NoError(t, err)
assert.True(t, fgs[gatewayFeature])
assert.True(t, fgs[GatewayFeature])

t.Log("configuring several invalid feature gates options")
featureGates = map[string]bool{"invalidGateway": true}

t.Log("verifying feature gates setup results when invalid feature gates options are present")
_, err = setupFeatureGates(setupLog, featureGates)
_, err = Setup(setupLog, featureGates)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalidGateway is not a valid feature")
}
42 changes: 4 additions & 38 deletions internal/manager/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,23 @@ import (

"github.com/blang/semver/v4"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
knativev1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/manager"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"

"github.com/kong/kubernetes-ingress-controller/v2/internal/adminapi"
"github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway"
"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/sendconfig"
"github.com/kong/kubernetes-ingress-controller/v2/internal/konnect"
"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/featuregates"
"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/metadata"
mgrutils "github.com/kong/kubernetes-ingress-controller/v2/internal/manager/utils"
"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/utils/kongconfig"
"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
"github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object/status"
"github.com/kong/kubernetes-ingress-controller/v2/internal/versions"
konghqcomv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
konghqcomv1alpha1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1alpha1"
configurationv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1"
)

// -----------------------------------------------------------------------------
Expand All @@ -49,7 +43,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic, d
gateway.SetControllerName(gatewayv1beta1.GatewayController(c.GatewayAPIControllerName))

setupLog.Info("getting enabled options and features")
featureGates, err := setupFeatureGates(setupLog, c.FeatureGates)
featureGates, err := featuregates.Setup(setupLog, c.FeatureGates)
if err != nil {
return fmt.Errorf("failed to configure feature gates: %w", err)
}
Expand Down Expand Up @@ -90,7 +84,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic, d
kongConfig.Init(ctx, setupLog, initialKongClients)

setupLog.Info("configuring and building the controller manager")
controllerOpts, err := setupControllerOptions(setupLog, c, dbMode)
controllerOpts, err := setupControllerOptions(setupLog, c, dbMode, featureGates)
if err != nil {
return fmt.Errorf("unable to setup controller options: %w", err)
}
Expand Down Expand Up @@ -144,7 +138,7 @@ func Run(ctx context.Context, c *Config, diagnostic util.ConfigDumpDiagnostic, d
return fmt.Errorf("unable to initialize dataplane synchronizer: %w", err)
}

if enabled, ok := featureGates[combinedRoutesFeature]; ok && enabled {
if enabled, ok := featureGates[featuregates.CombinedRoutesFeature]; ok && enabled {
dataplaneClient.EnableCombinedServiceRoutes()
setupLog.Info("combined routes mode has been enabled")
}
Expand Down Expand Up @@ -243,31 +237,3 @@ func readyzHandler(mgr manager.Manager, dataplaneSynchronizer IsReady) func(*htt
return nil
}
}

func getScheme() (*runtime.Scheme, error) {
scheme := runtime.NewScheme()

if err := clientgoscheme.AddToScheme(scheme); err != nil {
return nil, err
}
if err := konghqcomv1.AddToScheme(scheme); err != nil {
return nil, err
}
if err := konghqcomv1alpha1.AddToScheme(scheme); err != nil {
return nil, err
}
if err := configurationv1beta1.AddToScheme(scheme); err != nil {
return nil, err
}
if err := knativev1alpha1.AddToScheme(scheme); err != nil {
return nil, err
}
if err := gatewayv1alpha2.Install(scheme); err != nil {
return nil, err
}
if err := gatewayv1beta1.Install(scheme); err != nil {
return nil, err
}

return scheme, nil
}
53 changes: 53 additions & 0 deletions internal/manager/scheme/scheme.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package scheme

import (
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
knativev1alpha1 "knative.dev/networking/pkg/apis/networking/v1alpha1"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"

"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/featuregates"
konghqcomv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
konghqcomv1alpha1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1alpha1"
configurationv1beta1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1beta1"
)

// Get returns the scheme for the manager, enabling all the default schemes and
// those that were enabled via the feature flags.
func Get(fg map[string]bool) (*runtime.Scheme, error) {
scheme := runtime.NewScheme()

if err := clientgoscheme.AddToScheme(scheme); err != nil {
return nil, err
}

if err := konghqcomv1.AddToScheme(scheme); err != nil {
return nil, err
}
if err := konghqcomv1alpha1.AddToScheme(scheme); err != nil {
return nil, err
}
if err := configurationv1beta1.AddToScheme(scheme); err != nil {
return nil, err
}

if v, ok := fg[featuregates.KnativeFeature]; ok && v {
if err := knativev1alpha1.AddToScheme(scheme); err != nil {
return nil, err
}
}

if v, ok := fg[featuregates.GatewayAlphaFeature]; ok && v {
if err := gatewayv1alpha2.Install(scheme); err != nil {
return nil, err
}
}
if v, ok := fg[featuregates.GatewayFeature]; ok && v {
if err := gatewayv1beta1.Install(scheme); err != nil {
return nil, err
}
}

return scheme, nil
}
5 changes: 3 additions & 2 deletions internal/manager/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/kong/kubernetes-ingress-controller/v2/internal/adminapi"
"github.com/kong/kubernetes-ingress-controller/v2/internal/admission"
"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
"github.com/kong/kubernetes-ingress-controller/v2/internal/manager/scheme"
"github.com/kong/kubernetes-ingress-controller/v2/internal/util"
)

Expand Down Expand Up @@ -52,9 +53,9 @@ func SetupLoggers(c *Config, output io.Writer) (logrus.FieldLogger, logr.Logger,
return deprecatedLogger, logger, nil
}

func setupControllerOptions(logger logr.Logger, c *Config, dbmode string) (ctrl.Options, error) {
func setupControllerOptions(logger logr.Logger, c *Config, dbmode string, featureGates map[string]bool) (ctrl.Options, error) {
logger.Info("building the manager runtime scheme and loading apis into the scheme")
scheme, err := getScheme()
scheme, err := scheme.Get(featureGates)
if err != nil {
return ctrl.Options{}, err
}
Expand Down

0 comments on commit 1218fd1

Please sign in to comment.