Skip to content

Commit

Permalink
Merge pull request #126 from saschagrunert/retry-api
Browse files Browse the repository at this point in the history
Use `go-retry` for HTTP retry
  • Loading branch information
k8s-ci-robot authored Jan 31, 2025
2 parents de44574 + 6676419 commit 196ffaa
Show file tree
Hide file tree
Showing 5 changed files with 226 additions and 116 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module sigs.k8s.io/release-utils
go 1.23

require (
github.com/avast/retry-go/v4 v4.6.0
github.com/blang/semver/v4 v4.0.0
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4=
github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
Expand Down
102 changes: 61 additions & 41 deletions http/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ import (
"io"
"math"
"net/http"
"net/url"
"sync"
"time"

"github.com/avast/retry-go/v4"
"github.com/nozzle/throttler"
"github.com/sirupsen/logrus"
)
Expand Down Expand Up @@ -59,7 +61,8 @@ type agentOptions struct {
FailOnHTTPError bool // Set to true to fail on HTTP Status > 299
Retries uint // Number of times to retry when errors happen
Timeout time.Duration // Timeout when fetching URLs
MaxWaitTime time.Duration // Max waiting time when backing off
WaitTime time.Duration // Initial wait time for backing off on retry
MaxWaitTime time.Duration // Max waiting time when backing off on retry
PostContentType string // Content type to send when posting data
MaxParallel uint // Maximum number of parallel requests when requesting groups
}
Expand All @@ -76,6 +79,7 @@ var defaultAgentOptions = &agentOptions{
FailOnHTTPError: true,
Retries: 3,
Timeout: 3 * time.Second,
WaitTime: 2 * time.Second,
MaxWaitTime: 60 * time.Second,
PostContentType: defaultPostContentType,
MaxParallel: 5,
Expand Down Expand Up @@ -108,6 +112,20 @@ func (a *Agent) WithRetries(retries uint) *Agent {
return a
}

// WithWaitTime sets the initial wait time for request retry.
func (a *Agent) WithWaitTime(time time.Duration) *Agent {
a.options.WaitTime = time

return a
}

// WithMaxWaitTime sets the maximum wait time for request retry.
func (a *Agent) WithMaxWaitTime(time time.Duration) *Agent {
a.options.MaxWaitTime = time

return a
}

// WithFailOnHTTPError determines if the agent fails on HTTP errors (HTTP status not in 200s).
func (a *Agent) WithFailOnHTTPError(flag bool) *Agent {
a.options.FailOnHTTPError = flag
Expand Down Expand Up @@ -145,28 +163,9 @@ func (a *Agent) Get(url string) (content []byte, err error) {
func (a *Agent) GetRequest(url string) (response *http.Response, err error) {
logrus.Debugf("Sending GET request to %s", url)

var try uint

for {
response, err = a.AgentImplementation.SendGetRequest(a.Client(), url)
try++

if err == nil || try >= a.options.Retries {
return response, err
}
// Do exponential backoff...
waitTime := math.Pow(2, float64(try))
// ... but wait no more than 1 min
if waitTime > 60 {
waitTime = a.options.MaxWaitTime.Seconds()
}

logrus.Errorf(
"Error getting URL (will retry %d more times in %.0f secs): %s",
a.options.Retries-try, waitTime, err.Error(),
)
time.Sleep(time.Duration(waitTime) * time.Second)
}
return a.retryRequest(func() (*http.Response, error) {
return a.AgentImplementation.SendGetRequest(a.Client(), url)
})
}

// Post returns the body of a POST request.
Expand All @@ -184,28 +183,49 @@ func (a *Agent) Post(url string, postData []byte) (content []byte, err error) {
func (a *Agent) PostRequest(url string, postData []byte) (response *http.Response, err error) {
logrus.Debugf("Sending POST request to %s", url)

var try uint

for {
response, err = a.AgentImplementation.SendPostRequest(a.Client(), url, postData, a.options.PostContentType)
try++
return a.retryRequest(func() (*http.Response, error) {
return a.AgentImplementation.SendPostRequest(a.Client(), url, postData, a.options.PostContentType)
})
}

if err == nil || try >= a.options.Retries {
return response, err
}
// Do exponential backoff...
waitTime := math.Pow(2, float64(try))
// ... but wait no more than 1 min
if waitTime > 60 {
waitTime = a.options.MaxWaitTime.Seconds()
func (a *Agent) retryRequest(do func() (*http.Response, error)) (response *http.Response, err error) {
err = retry.Do(func() error {
//nolint:bodyclose // The API consumer should close the body
response, err = do()
if retryErr := shouldRetry(response, err); retryErr != nil {
return retryErr
}

logrus.Errorf(
"Error getting URL (will retry %d more times in %.0f secs): %s",
a.options.Retries-try, waitTime, err.Error(),
)
time.Sleep(time.Duration(waitTime) * time.Second)
return nil
},
retry.Attempts(a.options.Retries),
retry.Delay(a.options.WaitTime),
retry.MaxDelay(a.options.MaxWaitTime),
retry.DelayType(retry.BackOffDelay),
retry.OnRetry(func(attempt uint, err error) {
logrus.Errorf("Unable to do request (attempt %d/%d): %v", attempt+1, a.options.Retries, err)
}),
)

return response, err
}

func shouldRetry(resp *http.Response, err error) error {
urlErr := &url.Error{}
if err != nil && errors.As(err, &urlErr) {
return err
}

if resp.StatusCode == http.StatusTooManyRequests {
return fmt.Errorf("retry %d: %s", resp.StatusCode, resp.Status)
}

if resp.StatusCode == 0 || (resp.StatusCode >= 500 &&
resp.StatusCode != http.StatusNotImplemented) {
return fmt.Errorf("retry unexpected HTTP status %d: %s", resp.StatusCode, resp.Status)
}

return nil
}

// Head returns the body of a HEAD request.
Expand Down
162 changes: 162 additions & 0 deletions http/agent_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http_test

import (
"errors"
"net/http"
"net/url"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

rhttp "sigs.k8s.io/release-utils/http"
"sigs.k8s.io/release-utils/http/httpfakes"
)

func TestGetRequest(t *testing.T) {
for _, tc := range map[string]struct {
prepare func(*httpfakes.FakeAgentImplementation)
assert func(*http.Response, error)
}{
"should succeed": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendGetRequestReturns(&http.Response{StatusCode: http.StatusOK}, nil)
},
assert: func(response *http.Response, err error) {
require.NoError(t, err)
assert.Equal(t, http.StatusOK, response.StatusCode)
},
},
"should succeed on retry": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendGetRequestReturnsOnCall(0, &http.Response{StatusCode: http.StatusInternalServerError}, nil)
mock.SendGetRequestReturnsOnCall(1, &http.Response{StatusCode: http.StatusOK}, nil)
},
assert: func(response *http.Response, err error) {
require.NoError(t, err)
assert.Equal(t, http.StatusOK, response.StatusCode)
},
},
"should retry on internal server error": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendGetRequestReturns(&http.Response{StatusCode: http.StatusInternalServerError}, nil)
},
assert: func(response *http.Response, err error) {
require.Error(t, err)
assert.NotNil(t, response)
},
},
"should retry on too many requests": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendGetRequestReturns(&http.Response{StatusCode: http.StatusTooManyRequests}, nil)
},
assert: func(response *http.Response, err error) {
require.Error(t, err)
assert.NotNil(t, response)
},
},
"should retry on URL error": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendGetRequestReturns(nil, &url.Error{Err: errors.New("test")})
},
assert: func(response *http.Response, err error) {
require.Error(t, err)
require.Contains(t, err.Error(), "test")
assert.Nil(t, response)
},
},
} {
agent := rhttp.NewAgent().WithWaitTime(0)
mock := &httpfakes.FakeAgentImplementation{}
agent.SetImplementation(mock)

if tc.prepare != nil {
tc.prepare(mock)
}

//nolint:bodyclose // no need to close for mocked tests
tc.assert(agent.GetRequest(""))
}
}

func TestPostRequest(t *testing.T) {
for _, tc := range map[string]struct {
prepare func(*httpfakes.FakeAgentImplementation)
assert func(*http.Response, error)
}{
"should succeed": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendPostRequestReturns(&http.Response{StatusCode: http.StatusOK}, nil)
},
assert: func(response *http.Response, err error) {
require.NoError(t, err)
assert.Equal(t, http.StatusOK, response.StatusCode)
},
},
"should succeed on retry": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendPostRequestReturnsOnCall(0, &http.Response{StatusCode: http.StatusInternalServerError}, nil)
mock.SendPostRequestReturnsOnCall(1, &http.Response{StatusCode: http.StatusOK}, nil)
},
assert: func(response *http.Response, err error) {
require.NoError(t, err)
assert.Equal(t, http.StatusOK, response.StatusCode)
},
},
"should retry on internal server error": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendPostRequestReturns(&http.Response{StatusCode: http.StatusInternalServerError}, nil)
},
assert: func(response *http.Response, err error) {
require.Error(t, err)
assert.NotNil(t, response)
},
},
"should retry on too many requests": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendPostRequestReturns(&http.Response{StatusCode: http.StatusTooManyRequests}, nil)
},
assert: func(response *http.Response, err error) {
require.Error(t, err)
assert.NotNil(t, response)
},
},
"should retry on URL error": {
prepare: func(mock *httpfakes.FakeAgentImplementation) {
mock.SendPostRequestReturns(nil, &url.Error{Err: errors.New("test")})
},
assert: func(response *http.Response, err error) {
require.Error(t, err)
require.Contains(t, err.Error(), "test")
assert.Nil(t, response)
},
},
} {
agent := rhttp.NewAgent().WithWaitTime(0)
mock := &httpfakes.FakeAgentImplementation{}
agent.SetImplementation(mock)

if tc.prepare != nil {
tc.prepare(mock)
}

//nolint:bodyclose // no need to close for mocked tests
tc.assert(agent.PostRequest("", nil))
}
}
Loading

0 comments on commit 196ffaa

Please sign in to comment.