Skip to content

Commit

Permalink
Add rapid connection disconnection test
Browse files Browse the repository at this point in the history
  • Loading branch information
peterargue committed Feb 1, 2024
1 parent d99cfc2 commit ce48b5c
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 2 deletions.
101 changes: 100 additions & 1 deletion engine/access/rpc/connection/connection_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package connection

import (
"context"
"crypto/rand"
"fmt"
"math/big"
"net"
"sync"
"testing"
Expand Down Expand Up @@ -558,7 +560,7 @@ func TestExecutionNodeClientClosedGracefully(t *testing.T) {
ctx := context.Background()

// Generate random number of requests
nofRequests := rapid.IntRange(10, 100).Draw(tt, "nofRequests").(int)
nofRequests := rapid.IntRange(10, 100).Draw(tt, "nofRequests")
reqCompleted := atomic.NewUint64(0)

var waitGroup sync.WaitGroup
Expand Down Expand Up @@ -712,6 +714,103 @@ func TestEvictingCacheClients(t *testing.T) {
wg.Wait() // wait until the move test routine is done
}

func TestConcurrentConnections(t *testing.T) {
logger := unittest.Logger()
metrics := metrics.NewNoopCollector()

// Add createExecNode function to recreate it each time for rapid test
createExecNode := func() (*executionNode, func()) {
en := new(executionNode)
en.start(t)
return en, func() {
en.stop(t)
}
}

// setup the handler mock
req := &execution.PingRequest{}
resp := &execution.PingResponse{}

// Note: rapid will randomly fail with an error: "group did not use any data from bitstream"
// See https://github.com/flyingmutant/rapid/issues/65
rapid.Check(t, func(tt *rapid.T) {
en, closer := createExecNode()
defer closer()

responsesSent := atomic.NewInt32(0)
en.handler.
On("Ping", testifymock.Anything, req).
Return(func(_ context.Context, _ *execution.PingRequest) (*execution.PingResponse, error) {
sleepMicro := rapid.UintRange(100, 10_000).Draw(tt, "s")
time.Sleep(time.Duration(sleepMicro) * time.Microsecond)

// randomly fail ~25% of the time to test that client connection and reuse logic
// handles concurrent connect/disconnects
fail, err := rand.Int(rand.Reader, big.NewInt(4))
require.NoError(tt, err)

if fail.Uint64()%4 == 0 {
err = status.Errorf(codes.Unavailable, "random error")
}

responsesSent.Inc()
return resp, err
})

connectionCache, err := NewCache(logger, metrics, 1)
require.NoError(tt, err)

connectionFactory := &ConnectionFactoryImpl{
ExecutionGRPCPort: en.port,
ExecutionNodeGRPCTimeout: time.Second,
AccessMetrics: metrics,
Manager: NewManager(
logger,
metrics,
connectionCache,
0,
CircuitBreakerConfig{},
grpcutils.NoCompressor,
),
}

clientAddress := en.listener.Addr().String()

ctx := context.Background()

// Generate random number of requests
requestCount := rapid.IntRange(50, 1000).Draw(tt, "r")

var wg sync.WaitGroup
wg.Add(requestCount)

for i := 0; i < requestCount; i++ {
go func() {
defer wg.Done()

client, _, err := connectionFactory.GetExecutionAPIClient(clientAddress)
require.NoError(tt, err)

_, err = client.Ping(ctx, req)

if err != nil {
// Note: for some reason, when Unavailable is returned, the error message is
// changed to "the connection to 127.0.0.1:57753 was closed". Other error codes
// preserve the message.
require.Equalf(tt, codes.Unavailable, status.Code(err), "unexpected error: %v", err)
}
}()
}
wg.Wait()

// the grpc client seems to throttle requests to servers that return Unavailable, so not
// all of the requests make it through to the backend every test. Requiring that at least 1
// request is handled for these cases, but all should be handled in most runs.
assert.LessOrEqual(tt, responsesSent.Load(), int32(requestCount))
assert.Greater(tt, responsesSent.Load(), int32(0))
})
}

var successCodes = []codes.Code{
codes.Canceled,
codes.InvalidArgument,
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ require (
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0
google.golang.org/protobuf v1.31.0
gotest.tools v2.2.0+incompatible
pgregory.net/rapid v0.4.7
pgregory.net/rapid v1.1.0
)

require (
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,7 @@ github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvD
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo=
github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
Expand Down Expand Up @@ -857,6 +858,7 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
Expand Down Expand Up @@ -1242,6 +1244,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
Expand Down Expand Up @@ -2477,6 +2480,8 @@ nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g=
pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
Expand Down

0 comments on commit ce48b5c

Please sign in to comment.