Explicitly set a exponential backoff rate limiter for controller config

The rate limiter base delay and max delay values can be configured using
flags. The default values are 500ms for base delay and 15min for max delay.

controllers/options.go

@@ -1,10 +1,15 @@
 package controllers
 
-import "errors"
+import (
+	"errors"
+
+	"k8s.io/client-go/util/workqueue"
+)
 
 // ControllerConfig is the configuration for cluster and machine controllers
 type ControllerConfig struct {
 	MaxConcurrentReconciles int
+	RateLimiter             workqueue.RateLimiter
 }
 
 // ControllerConfigOpts is a function that can be used to configure the controller config
@@ -20,3 +25,14 @@ func WithMaxConcurrentReconciles(max int) ControllerConfigOpts {
 		return nil
 	}
 }
+
+// WithRateLimiter sets the rate limiter for the controller
+func WithRateLimiter(rateLimiter workqueue.RateLimiter) ControllerConfigOpts {
+	return func(c *ControllerConfig) error {
+		if rateLimiter == nil {
+			return errors.New("rate limiter cannot be nil")
+		}
+		c.RateLimiter = rateLimiter
+		return nil
+	}
+}

main.go

@@ -35,6 +35,7 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
 	capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	capiflags "sigs.k8s.io/cluster-api/util/flags"
@@ -75,8 +76,13 @@ type managerConfig struct {
 	concurrentReconcilesNutanixMachine int
 	diagnosticsOptions                 capiflags.DiagnosticsOptions
 
-	logger     logr.Logger
-	restConfig *rest.Config
+	logger      logr.Logger
+	restConfig  *rest.Config
+	rateLimiter workqueue.RateLimiter
+}
+
+func rateLimiter(baseDelay, maxDelay time.Duration) workqueue.RateLimiter {
+	return workqueue.NewItemExponentialFailureRateLimiter(baseDelay, maxDelay)
 }
 
 func parseFlags(config *managerConfig) {
@@ -88,6 +94,10 @@ func parseFlags(config *managerConfig) {
 	pflag.IntVar(&maxConcurrentReconciles, "max-concurrent-reconciles", defaultMaxConcurrentReconciles,
 		"The maximum number of allowed, concurrent reconciles.")
 
+	var baseDelay, maxDelay time.Duration
+	pflag.DurationVar(&baseDelay, "rate-limiter-base-delay", 500*time.Millisecond, "The base delay for the rate limiter.")
+	pflag.DurationVar(&maxDelay, "rate-limiter-max-delay", 15*time.Minute, "The maximum delay for the rate limiter.")
+
 	opts := zap.Options{
 		TimeEncoder: zapcore.RFC3339TimeEncoder,
 	}
@@ -100,6 +110,7 @@ func parseFlags(config *managerConfig) {
 
 	config.concurrentReconcilesNutanixCluster = maxConcurrentReconciles
 	config.concurrentReconcilesNutanixMachine = maxConcurrentReconciles
+	config.rateLimiter = rateLimiter(baseDelay, maxDelay)
 }
 
 func setupLogger() logr.Logger {
@@ -189,6 +200,7 @@ func runManager(ctx context.Context, mgr manager.Manager, config *managerConfig)
 
 	clusterControllerOpts := []controllers.ControllerConfigOpts{
 		controllers.WithMaxConcurrentReconciles(config.concurrentReconcilesNutanixCluster),
+		controllers.WithRateLimiter(config.rateLimiter),
 	}
 
 	if err := setupNutanixClusterController(ctx, mgr, secretInformer, configMapInformer, clusterControllerOpts...); err != nil {
@@ -197,6 +209,7 @@ func runManager(ctx context.Context, mgr manager.Manager, config *managerConfig)
 
 	machineControllerOpts := []controllers.ControllerConfigOpts{
 		controllers.WithMaxConcurrentReconciles(config.concurrentReconcilesNutanixMachine),
+		controllers.WithRateLimiter(config.rateLimiter),
 	}
 
 	if err := setupNutanixMachineController(ctx, mgr, secretInformer, configMapInformer, machineControllerOpts...); err != nil {

main_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
@@ -125,6 +126,7 @@ func testRunManagerCommon(t *testing.T, ctrl *gomock.Controller) (*mockctlclient
 		concurrentReconcilesNutanixCluster: 1,
 		concurrentReconcilesNutanixMachine: 1,
 		restConfig:                         cfg,
+		rateLimiter:                        rateLimiter(500*time.Millisecond, 15*time.Minute),
 	}
 
 	client := mockctlclient.NewMockClient(ctrl)