Add configurable exponential backoff to Agent auto-auth

command/agent.go

@@ -575,6 +575,7 @@ func (c *AgentCommand) Run(args []string) int {
 			Logger:                       c.logger.Named("auth.handler"),
 			Client:                       c.client,
 			WrapTTL:                      config.AutoAuth.Method.WrapTTL,
+			MaxBackoff:                   config.AutoAuth.Method.MaxBackoff,
 			EnableReauthOnNewCredentials: config.AutoAuth.EnableReauthOnNewCredentials,
 			EnableTemplateTokenCh:        enableTokenCh,
 		})

command/agent/auth/auth.go

@@ -8,11 +8,16 @@ import (
 	"net/http"
 	"time"
 
-	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 )
 
+const (
+	initialBackoff    = 1 * time.Second
+	defaultMaxBackoff = 5 * time.Minute
+)
+
 // AuthMethod is the interface that auto-auth methods implement for the agent
 // to use.
 type AuthMethod interface {
@@ -48,6 +53,7 @@ type AuthHandler struct {
 	client                       *api.Client
 	random                       *rand.Rand
 	wrapTTL                      time.Duration
+	maxBackoff                   time.Duration
 	enableReauthOnNewCredentials bool
 	enableTemplateTokenCh        bool
 }
@@ -56,6 +62,7 @@ type AuthHandlerConfig struct {
 	Logger                       hclog.Logger
 	Client                       *api.Client
 	WrapTTL                      time.Duration
+	MaxBackoff                   time.Duration
 	Token                        string
 	EnableReauthOnNewCredentials bool
 	EnableTemplateTokenCh        bool
@@ -72,6 +79,7 @@ func NewAuthHandler(conf *AuthHandlerConfig) *AuthHandler {
 		client:                       conf.Client,
 		random:                       rand.New(rand.NewSource(int64(time.Now().Nanosecond()))),
 		wrapTTL:                      conf.WrapTTL,
+		maxBackoff:                   conf.MaxBackoff,
 		enableReauthOnNewCredentials: conf.EnableReauthOnNewCredentials,
 		enableTemplateTokenCh:        conf.EnableTemplateTokenCh,
 	}
@@ -91,6 +99,13 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 		return errors.New("auth handler: nil auth method")
 	}
 
+	backoff := initialBackoff
+	maxBackoff := defaultMaxBackoff
+
+	if ah.maxBackoff > 0 {
+		maxBackoff = ah.maxBackoff
+	}
+
 	ah.logger.Info("starting auth handler")
 	defer func() {
 		am.Shutdown()
@@ -130,8 +145,7 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 		default:
 		}
 
-		// Create a fresh backoff value
-		backoff := 2*time.Second + time.Duration(ah.random.Int63()%int64(time.Second*2)-int64(time.Second))
+		backoff = calculateBackoff(backoff, maxBackoff)
 
 		var clientToUse *api.Client
 		var err error
@@ -311,3 +325,16 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
 		}
 	}
 }
+
+// calculateBackoff determines a new backoff duration that is roughly twice
+// the previous value, capped to a max value, with a measure of randomness.
+func calculateBackoff(previous, max time.Duration) time.Duration {
+	maxBackoff := 2 * previous
+	if maxBackoff > max {
+		maxBackoff = max
+	}
+
+	// Trim a random amount (0-25%) off the doubled duration
+	trim := rand.Int63n(int64(maxBackoff) / 4)
+	return maxBackoff - time.Duration(trim)
+}

command/agent/auth/auth_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/builtin/credential/userpass"
 	vaulthttp "github.com/hashicorp/vault/http"
@@ -106,3 +106,42 @@ consumption:
 		}
 	}
 }
+
+func TestCalculateBackoff(t *testing.T) {
+	tests := []struct {
+		previous time.Duration
+		max      time.Duration
+		expMin   time.Duration
+		expMax   time.Duration
+	}{
+		{
+			1000 * time.Millisecond,
+			60000 * time.Millisecond,
+			1500 * time.Millisecond,
+			2000 * time.Millisecond,
+		},
+		{
+			1000 * time.Millisecond,
+			5000 * time.Millisecond,
+			1500 * time.Millisecond,
+			2000 * time.Millisecond,
+		},
+		{
+			4000 * time.Millisecond,
+			5000 * time.Millisecond,
+			3750 * time.Millisecond,
+			5000 * time.Millisecond,
+		},
+	}
+
+	for _, test := range tests {
+		for i := 0; i < 100; i++ {
+			backoff := calculateBackoff(test.previous, test.max)
+
+			// Verify that the new backoff is 75-100% of 2*previous, but <= than the max
+			if backoff < test.expMin || backoff > test.expMax {
+				t.Fatalf("expected backoff in range %v to %v, got: %v", test.expMin, test.expMax, backoff)
+			}
+		}
+	}
+}

command/agent/config/config.go

@@ -62,12 +62,14 @@ type AutoAuth struct {
 
 // Method represents the configuration for the authentication backend
 type Method struct {
-	Type       string
-	MountPath  string        `hcl:"mount_path"`
-	WrapTTLRaw interface{}   `hcl:"wrap_ttl"`
-	WrapTTL    time.Duration `hcl:"-"`
-	Namespace  string        `hcl:"namespace"`
-	Config     map[string]interface{}
+	Type          string
+	MountPath     string        `hcl:"mount_path"`
+	WrapTTLRaw    interface{}   `hcl:"wrap_ttl"`
+	WrapTTL       time.Duration `hcl:"-"`
+	MaxBackoffRaw interface{}   `hcl:"max_backoff"`
+	MaxBackoff    time.Duration `hcl:"-"`
+	Namespace     string        `hcl:"namespace"`
+	Config        map[string]interface{}
 }
 
 // Sink defines a location to write the authenticated token
@@ -358,6 +360,14 @@ func parseAutoAuth(result *Config, list *ast.ObjectList) error {
 		}
 	}
 
+	if result.AutoAuth.Method.MaxBackoffRaw != nil {
+		var err error
+		if result.AutoAuth.Method.MaxBackoff, err = parseutil.ParseDurationSecond(result.AutoAuth.Method.MaxBackoffRaw); err != nil {
+			return err
+		}
+		result.AutoAuth.Method.MaxBackoffRaw = nil
+	}
+
 	return nil
 }
 

command/agent/config/config_test.go

@@ -126,6 +126,7 @@ func TestLoadConfigFile(t *testing.T) {
 				Config: map[string]interface{}{
 					"role": "foobar",
 				},
+				MaxBackoff: 0,
 			},
 			Sinks: []*Sink{
 				{
@@ -178,9 +179,10 @@ func TestLoadConfigFile_Method_Wrapping(t *testing.T) {
 		},
 		AutoAuth: &AutoAuth{
 			Method: &Method{
-				Type:      "aws",
-				MountPath: "auth/aws",
-				WrapTTL:   5 * time.Minute,
+				Type:       "aws",
+				MountPath:  "auth/aws",
+				WrapTTL:    5 * time.Minute,
+				MaxBackoff: 2 * time.Minute,
 				Config: map[string]interface{}{
 					"role": "foobar",
 				},

command/agent/config/test-fixtures/config-method-wrapping.hcl

@@ -7,6 +7,7 @@ auto_auth {
 		config = {
 			role = "foobar"
 		}
+		max_backoff = "2m"
 	}
 
 	sink {