Adding retry for eni plugin to acquire mac address

pkg/utils/backoff.go

@@ -0,0 +1,83 @@
+// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package utils
+
+import (
+	"math"
+	"math/rand"
+	"sync"
+	"time"
+)
+
+// Backoff defines the interface for a backoff
+type Backoff interface {
+	Reset()
+	Duration() time.Duration
+}
+
+// SimpleBackoff implements a simple mechanism for backoff
+type SimpleBackoff struct {
+	current        time.Duration
+	start          time.Duration
+	max            time.Duration
+	jitterMultiple float64
+	multiple       float64
+	mu             sync.Mutex
+}
+
+// NewSimpleBackoff creates a Backoff which ranges from min to max increasing by
+// multiple each time.
+// It also adds (and yes, the jitter is always added, never
+// subtracted) a random amount of jitter up to jitterMultiple percent (that is,
+// jitterMultiple = 0.0 is no jitter, 0.15 is 15% added jitter). The total time
+// may exceed "max" when accounting for jitter, such that the absolute max is
+// max + max * jiterMultiple
+func NewSimpleBackoff(min, max time.Duration, jitterMultiple, multiple float64) *SimpleBackoff {
+	return &SimpleBackoff{
+		start:          min,
+		current:        min,
+		max:            max,
+		jitterMultiple: jitterMultiple,
+		multiple:       multiple,
+	}
+}
+
+// Duration returns the time a backoff should wait
+func (sb *SimpleBackoff) Duration() time.Duration {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+	ret := sb.current
+	sb.current = time.Duration(math.Min(float64(sb.max.Nanoseconds()), float64(float64(sb.current.Nanoseconds())*sb.multiple)))
+
+	return AddJitter(ret, time.Duration(int64(float64(ret)*sb.jitterMultiple)))
+}
+
+// Reset sets backoff to its initial status
+func (sb *SimpleBackoff) Reset() {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+	sb.current = sb.start
+}
+
+// AddJitter adds an amount of jitter between 0 and the given jitter to the
+// given duration
+func AddJitter(duration time.Duration, jitter time.Duration) time.Duration {
+	var randJitter int64
+	if jitter.Nanoseconds() == 0 {
+		randJitter = 0
+	} else {
+		randJitter = rand.Int63n(jitter.Nanoseconds())
+	}
+	return time.Duration(duration.Nanoseconds() + randJitter)
+}

pkg/utils/backoff_test.go

@@ -0,0 +1,51 @@
+// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package utils
+
+import (
+	"testing"
+	"time"
+)
+
+func TestSimpleBackoff(t *testing.T) {
+	sb := NewSimpleBackoff(10*time.Second, time.Minute, 0, 2)
+
+	for i := 0; i < 2; i++ {
+		duration := sb.Duration()
+		if duration.Nanoseconds() != 10*time.Second.Nanoseconds() {
+			t.Error("Initial duration incorrect. Got ", duration.Nanoseconds())
+		}
+
+		duration = sb.Duration()
+		if duration.Nanoseconds() != 20*time.Second.Nanoseconds() {
+			t.Error("Increase incorrect")
+		}
+		_ = sb.Duration() // 40s
+		duration = sb.Duration()
+		if duration.Nanoseconds() != 60*time.Second.Nanoseconds() {
+			t.Error("Didn't stop at maximum")
+		}
+		sb.Reset()
+		// loop to redo the above tests after resetting, they should be the same
+	}
+}
+
+func TestJitter(t *testing.T) {
+	for i := 0; i < 10; i++ {
+		duration := AddJitter(10*time.Second, 3*time.Second)
+		if duration < 10*time.Second || duration > 13*time.Second {
+			t.Error("Excessive amount of jitter", duration)
+		}
+	}
+}

pkg/utils/errors.go

@@ -0,0 +1,56 @@
+// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//	http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package utils
+
+// Retriable definies the interface for retriable object
+type Retriable interface {
+	Retry() bool
+}
+
+// DefaultRetriable is a simple struct that implements the Retriable
+type DefaultRetriable struct {
+	retry bool
+}
+
+// Retry returns whether should retry or not
+func (dr DefaultRetriable) Retry() bool {
+	return dr.retry
+}
+
+// NewRetriable creates a simple Retriable object
+func NewRetriable(retry bool) Retriable {
+	return DefaultRetriable{
+		retry: retry,
+	}
+}
+
+// RetriableError definies the interface for retriable error
+type RetriableError interface {
+	Retriable
+	error
+}
+
+// DefaultRetriableError is a simple struct that implements the RetriableError
+type DefaultRetriableError struct {
+	Retriable
+	error
+}
+
+// NewRetriableError creates a simple RetriableError object
+func NewRetriableError(retriable Retriable, err error) RetriableError {
+	return &DefaultRetriableError{
+		retriable,
+		err,
+	}
+}

pkg/utils/utils.go

@@ -14,7 +14,9 @@
 package utils
 
 import (
+	"context"
 	"reflect"
+	"time"
 )
 
 // ZeroOrNil checks if the passed in interface is empty
@@ -46,3 +48,36 @@ func ZeroOrNil(obj interface{}) bool {
 	}
 	return false
 }
+
+// RetryWithBackoff takes a Backoff and a function to call that returns an error
+// If the error is nil then the function will no longer be called
+// If the error is Retriable then that will be used to determine if it should be
+// retried
+func RetryWithBackoff(backoff Backoff, fn func() error) error {
+	return RetryWithBackoffCtx(context.Background(), backoff, fn)
+}
+
+// RetryWithBackoffCtx takes a context, a Backoff, and a function to call that returns an error
+// If the context is done, nil will be returned
+// If the error is nil then the function will no longer be called
+// If the error is Retriable then that will be used to determine if it should be
+// retried
+func RetryWithBackoffCtx(ctx context.Context, backoff Backoff, fn func() error) error {
+	var err error
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		err = fn()
+		retriableErr, isRetriableErr := err.(Retriable)
+		if err == nil || (isRetriableErr && !retriableErr.Retry()) {
+			return err
+		}
+
+		time.Sleep(backoff.Duration())
+	}
+	return err
+}

pkg/utils/utils_test.go

@@ -14,7 +14,10 @@
 package utils
 
 import (
+	"context"
+	"errors"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -53,3 +56,65 @@ func TestZeroOrNil(t *testing.T) {
 		})
 	}
 }
+
+func TestRetryWithBackoff(t *testing.T) {
+	t.Run("retries", func(t *testing.T) {
+		counter := 3
+		RetryWithBackoff(NewSimpleBackoff(10*time.Millisecond, 10*time.Millisecond, 0, 1), func() error {
+			if counter == 0 {
+				return nil
+			}
+			counter--
+			return errors.New("err")
+		})
+		assert.Equal(t, 0, counter, "Counter didn't go to 0; didn't get retried enough")
+	})
+
+	t.Run("no retries", func(t *testing.T) {
+		counter := 3
+		RetryWithBackoff(NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error {
+			counter--
+			return NewRetriableError(NewRetriable(false), errors.New("can't retry"))
+		})
+		assert.Equal(t, 2, counter, "Counter should only be operated once without retry")
+	})
+}
+
+func TestRetryWithBackoffCtx(t *testing.T) {
+	t.Run("retries", func(t *testing.T) {
+		counter := 3
+		RetryWithBackoffCtx(context.TODO(), NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error {
+			if counter == 0 {
+				return nil
+			}
+			counter--
+			return errors.New("err")
+		})
+		assert.Equal(t, 0, counter, "Counter didn't go to 0; didn't get retried enough")
+	})
+
+	t.Run("no retries", func(t *testing.T) {
+		counter := 3
+		ctx, cancel := context.WithCancel(context.TODO())
+		cancel()
+		err := RetryWithBackoffCtx(ctx, NewSimpleBackoff(10*time.Second, 20*time.Second, 0, 2), func() error {
+			counter--
+			return NewRetriableError(NewRetriable(false), errors.New("can't retry"))
+		})
+		assert.Equal(t, 3, counter, "Counter should not be operated with context canceled")
+		assert.Error(t, err)
+	})
+
+	t.Run("cancel context", func(t *testing.T) {
+		counter := 2
+		ctx, cancel := context.WithCancel(context.TODO())
+		RetryWithBackoffCtx(ctx, NewSimpleBackoff(100*time.Millisecond, 100*time.Millisecond, 0, 1), func() error {
+			counter--
+			if counter == 0 {
+				cancel()
+			}
+			return errors.New("err")
+		})
+		assert.Equal(t, 0, counter, "Counter not 0; went the wrong number of times")
+	})
+}