From 4e1bbac48d75f8ed32659a2eada8f165817c6e5c Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Fri, 14 Sep 2018 17:10:53 -0400 Subject: [PATCH 1/8] set TCP_USER_TIMEOUT socketopt See gRPC proposal A18. --- internal/transport/http2_client.go | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go index 904e790c466d..782f6c5c24de 100644 --- a/internal/transport/http2_client.go +++ b/internal/transport/http2_client.go @@ -19,13 +19,17 @@ package transport import ( + "errors" + "fmt" "io" "math" "net" + "runtime" "strconv" "strings" "sync" "sync/atomic" + "syscall" "time" "golang.org/x/net/context" @@ -137,6 +141,30 @@ func isTemporary(err error) bool { return true } +func setTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + // requires Linux kernel version >= 2.6.37 + if runtime.GOOS != "linux" { + return nil + } + + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + return errors.New("error in casting *net.Conn to *net.TCPConn") + } + file, err := tcpconn.File() + if err != nil { + return fmt.Errorf("error getting file for connection: %v", err) + } + const TCP_USER_TIMEOUT = 0x12 + err = syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_TCP, TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) + file.Close() + if err != nil { + return fmt.Errorf("error in setting priority option on socket: %v", err) + } + + return nil +} + // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 // and starts to receive messages on it. Non-nil error returns if construction // fails. @@ -252,6 +280,9 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, "") } if t.kp.Time != infinity { + if err = setTCPUserTimeout(t.conn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } t.keepaliveEnabled = true go t.keepalive() } From 3cfab976c020bff513cff031d4c726c2f69fa213 Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Fri, 14 Sep 2018 17:19:51 -0400 Subject: [PATCH 2/8] clean up error messages --- internal/transport/http2_client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go index 782f6c5c24de..9b0f6d662fa1 100644 --- a/internal/transport/http2_client.go +++ b/internal/transport/http2_client.go @@ -149,7 +149,7 @@ func setTCPUserTimeout(conn net.Conn, timeout time.Duration) error { tcpconn, ok := conn.(*net.TCPConn) if !ok { - return errors.New("error in casting *net.Conn to *net.TCPConn") + return errors.New("error casting *net.Conn to *net.TCPConn") } file, err := tcpconn.File() if err != nil { @@ -159,7 +159,7 @@ func setTCPUserTimeout(conn net.Conn, timeout time.Duration) error { err = syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_TCP, TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) file.Close() if err != nil { - return fmt.Errorf("error in setting priority option on socket: %v", err) + return fmt.Errorf("error setting option on socket: %v", err) } return nil From 44836e0ae600721224e9bcaad0d734180bdce290 Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Fri, 14 Sep 2018 18:09:40 -0400 Subject: [PATCH 3/8] move function to syscall --- internal/syscall/syscall_linux.go | 23 +++++++++++++++++++++ internal/syscall/syscall_nonlinux.go | 12 ++++++++++- internal/transport/http2_client.go | 31 ++-------------------------- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go index 87bc65a19674..5b7bd7c91aba 100644 --- a/internal/syscall/syscall_linux.go +++ b/internal/syscall/syscall_linux.go @@ -23,7 +23,11 @@ package syscall import ( + "errors" + "fmt" + "net" "syscall" + "time" "golang.org/x/sys/unix" "google.golang.org/grpc/grpclog" @@ -65,3 +69,22 @@ func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { return uTimeElapsed, sTimeElapsed } + +// SetTCPUserTimeout sets the TCP user timeout on a connection's socket +func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + return errors.New("error casting *net.Conn to *net.TCPConn") + } + file, err := tcpconn.File() + if err != nil { + return fmt.Errorf("error getting file for connection: %v", err) + } + err = syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) + file.Close() + if err != nil { + return fmt.Errorf("error setting option on socket: %v", err) + } + + return nil +} diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go index 16a5c3fe45cb..8e12c5a7239b 100644 --- a/internal/syscall/syscall_nonlinux.go +++ b/internal/syscall/syscall_nonlinux.go @@ -20,7 +20,12 @@ package syscall -import "google.golang.org/grpc/grpclog" +import ( + "net" + "time" + + "google.golang.org/grpc/grpclog" +) func init() { grpclog.Info("CPU time info is unavailable on non-linux or appengine environment.") @@ -45,3 +50,8 @@ func GetRusage() (rusage *Rusage) { func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { return 0, 0 } + +// SetTCPUserTimeout is a no-op function under non-linux or appengine environments +func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + return nil +} diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go index 9b0f6d662fa1..93334f35e0a0 100644 --- a/internal/transport/http2_client.go +++ b/internal/transport/http2_client.go @@ -19,17 +19,13 @@ package transport import ( - "errors" - "fmt" "io" "math" "net" - "runtime" "strconv" "strings" "sync" "sync/atomic" - "syscall" "time" "golang.org/x/net/context" @@ -39,6 +35,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -141,30 +138,6 @@ func isTemporary(err error) bool { return true } -func setTCPUserTimeout(conn net.Conn, timeout time.Duration) error { - // requires Linux kernel version >= 2.6.37 - if runtime.GOOS != "linux" { - return nil - } - - tcpconn, ok := conn.(*net.TCPConn) - if !ok { - return errors.New("error casting *net.Conn to *net.TCPConn") - } - file, err := tcpconn.File() - if err != nil { - return fmt.Errorf("error getting file for connection: %v", err) - } - const TCP_USER_TIMEOUT = 0x12 - err = syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_TCP, TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) - file.Close() - if err != nil { - return fmt.Errorf("error setting option on socket: %v", err) - } - - return nil -} - // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 // and starts to receive messages on it. Non-nil error returns if construction // fails. @@ -280,7 +253,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, "") } if t.kp.Time != infinity { - if err = setTCPUserTimeout(t.conn, kp.Timeout); err != nil { + if err = syscall.SetTCPUserTimeout(t.conn, kp.Timeout); err != nil { return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) } t.keepaliveEnabled = true From ab5224abd5d6a21cf22224fe38e458be928e3849 Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Fri, 14 Sep 2018 20:10:59 -0400 Subject: [PATCH 4/8] move keepaliveEnabled check and SetTCPUserTimeout before handshake SetTCPUserTimeout is predicated upon keepaliveEnabled. SetTCPUserTimeout requires access to the underlying TCPConn and must be called before handshaking. --- internal/transport/http2_client.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go index 93334f35e0a0..e803f5fcc18e 100644 --- a/internal/transport/http2_client.go +++ b/internal/transport/http2_client.go @@ -163,6 +163,21 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne conn.Close() } }(conn) + kp := opts.KeepaliveParams + // Validate keepalive parameters. + if kp.Time == 0 { + kp.Time = defaultClientKeepaliveTime + } + if kp.Timeout == 0 { + kp.Timeout = defaultClientKeepaliveTimeout + } + keepaliveEnabled := false + if kp.Time != infinity { + if err = syscall.SetTCPUserTimeout(conn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } + keepaliveEnabled = true + } var ( isSecure bool authInfo credentials.AuthInfo @@ -175,14 +190,6 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne } isSecure = true } - kp := opts.KeepaliveParams - // Validate keepalive parameters. - if kp.Time == 0 { - kp.Time = defaultClientKeepaliveTime - } - if kp.Timeout == 0 { - kp.Timeout = defaultClientKeepaliveTimeout - } dynamicWindow := true icwz := int32(initialWindowSize) if opts.InitialConnWindowSize >= defaultWindowSize { @@ -224,6 +231,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne streamQuota: defaultMaxStreamsClient, streamsQuotaAvailable: make(chan struct{}, 1), czData: new(channelzData), + keepaliveEnabled: keepaliveEnabled, } t.controlBuf = newControlBuffer(t.ctxDone) if opts.InitialWindowSize >= defaultWindowSize { @@ -252,11 +260,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne if channelz.IsOn() { t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, "") } - if t.kp.Time != infinity { - if err = syscall.SetTCPUserTimeout(t.conn, kp.Timeout); err != nil { - return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) - } - t.keepaliveEnabled = true + if t.keepaliveEnabled { go t.keepalive() } // Start the reader goroutine for incoming message. Each transport has From a5123f87211eb83c763b7c23a7e62cff44599088 Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Fri, 14 Sep 2018 20:12:24 -0400 Subject: [PATCH 5/8] exit early unless conn is TCP --- internal/syscall/syscall_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go index 5b7bd7c91aba..c69c1f3898ae 100644 --- a/internal/syscall/syscall_linux.go +++ b/internal/syscall/syscall_linux.go @@ -23,7 +23,6 @@ package syscall import ( - "errors" "fmt" "net" "syscall" @@ -74,7 +73,8 @@ func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { tcpconn, ok := conn.(*net.TCPConn) if !ok { - return errors.New("error casting *net.Conn to *net.TCPConn") + // not a TCP connection. exit early + return nil } file, err := tcpconn.File() if err != nil { From 60d7792d55f8d2734e33b8fcde4e2375c4935267 Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Tue, 16 Oct 2018 22:51:56 -0400 Subject: [PATCH 6/8] switch from File() to SyscallConn() and add test net.TCPConn.File() was causing tests to time out. The docs indicate File() returns a different file descriptor than the connection's. --- internal/syscall/syscall_linux.go | 35 ++++++++++++++++--- internal/syscall/syscall_nonlinux.go | 8 +++++ internal/transport/transport_test.go | 51 ++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go index c69c1f3898ae..8cc864179577 100644 --- a/internal/syscall/syscall_linux.go +++ b/internal/syscall/syscall_linux.go @@ -23,6 +23,7 @@ package syscall import ( + "errors" "fmt" "net" "syscall" @@ -32,6 +33,8 @@ import ( "google.golang.org/grpc/grpclog" ) +var GetTCPUserTimeoutNoopError = errors.New("placeholder error") + // GetCPUTime returns the how much CPU time has passed since the start of this process. func GetCPUTime() int64 { var ts unix.Timespec @@ -76,15 +79,39 @@ func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { // not a TCP connection. exit early return nil } - file, err := tcpconn.File() + rawConn, err := tcpconn.SyscallConn() if err != nil { - return fmt.Errorf("error getting file for connection: %v", err) + return fmt.Errorf("error getting raw connection: %v", err) } - err = syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) - file.Close() + err = rawConn.Control(func(fd uintptr) { + err = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) + }) if err != nil { return fmt.Errorf("error setting option on socket: %v", err) } return nil } + +// GetTCPUserTimeout gets the TCP user timeout on a connection's socket +func GetTCPUserTimeout(conn net.Conn) (opt int, err error) { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + err = fmt.Errorf("conn is not *net.TCPConn. got %T", conn) + return + } + rawConn, err := tcpconn.SyscallConn() + if err != nil { + err = fmt.Errorf("error getting raw connection: %v", err) + return + } + err = rawConn.Control(func(fd uintptr) { + opt, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT) + }) + if err != nil { + err = fmt.Errorf("error getting option on socket: %v", err) + return + } + + return +} diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go index 8e12c5a7239b..3c7a9eb7aff8 100644 --- a/internal/syscall/syscall_nonlinux.go +++ b/internal/syscall/syscall_nonlinux.go @@ -21,12 +21,15 @@ package syscall import ( + "errors" "net" "time" "google.golang.org/grpc/grpclog" ) +var GetTCPUserTimeoutNoopError = errors.New("GetTCPUserTimeout is a no-op on non-linux or appengine environments") + func init() { grpclog.Info("CPU time info is unavailable on non-linux or appengine environment.") } @@ -55,3 +58,8 @@ func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { return nil } + +// GetTCPUserTimeout is a no-op function under non-linux or appengine environments +func GetTCPUserTimeout(conn net.Conn) (int, error) { + return 0, GetTCPUserTimeoutNoopError +} diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go index 45e29be8cfb6..35b6351d9dbd 100644 --- a/internal/transport/transport_test.go +++ b/internal/transport/transport_test.go @@ -41,6 +41,7 @@ import ( "golang.org/x/net/http2/hpack" "google.golang.org/grpc/codes" "google.golang.org/grpc/internal/leakcheck" + "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/status" ) @@ -2317,3 +2318,53 @@ func TestHeaderTblSize(t *testing.T) { t.Fatalf("expected len(limits) = 2 within 10s, got != 2") } } + +func TestTCPUserTimeout(t *testing.T) { + tests := []struct { + time time.Duration + timeout time.Duration + }{ + { + 10 * time.Second, + 10 * time.Second, + }, + { + 0, + 0, + }, + } + for _, tt := range tests { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen. Err: %v", err) + } + defer lis.Close() + // TODO(deklerk): we can `defer cancel()` here after we drop Go 1.6 support. Until then, + // doing a `defer cancel()` could cause the dialer to become broken: + // https://github.com/golang/go/issues/15078, https://github.com/golang/go/issues/15035 + connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) + client, err := newHTTP2Client(connectCtx, context.Background(), TargetInfo{Addr: lis.Addr().String()}, ConnectOptions{ + KeepaliveParams: keepalive.ClientParameters{ + Time: tt.time, + Timeout: tt.timeout, + }, + }, func() {}, func(GoAwayReason) {}, func() {}) + if err != nil { + cancel() // Do not cancel in success path. + t.Fatalf("error creating client: %v", err) + } + defer client.Close() + + opt, err := syscall.GetTCPUserTimeout(client.conn) + if err != nil { + if err == syscall.GetTCPUserTimeoutNoopError { + t.Skipf("skipping test on unsupported environment: %v", err) + } + t.Fatalf("GetTCPUserTimeout error: %v", err) + } + if timeoutMS := int(tt.timeout / time.Millisecond); timeoutMS != opt { + t.Fatalf("wrong TCP_USER_TIMEOUT set on conn. expected %d. got %d", + timeoutMS, opt) + } + } +} From e888557a3775e03c81db4fdabc0527fda1fd89a4 Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Wed, 17 Oct 2018 20:33:38 -0400 Subject: [PATCH 7/8] use negative value instead of error to indicate unsupported env A negative socket option int should never occur in actual operation. Returning a nil error is simpler and more consistent with existing functions in syscall. --- internal/syscall/syscall_linux.go | 3 --- internal/syscall/syscall_nonlinux.go | 6 ++---- internal/transport/transport_test.go | 6 +++--- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go index 8cc864179577..09a35ddf325e 100644 --- a/internal/syscall/syscall_linux.go +++ b/internal/syscall/syscall_linux.go @@ -23,7 +23,6 @@ package syscall import ( - "errors" "fmt" "net" "syscall" @@ -33,8 +32,6 @@ import ( "google.golang.org/grpc/grpclog" ) -var GetTCPUserTimeoutNoopError = errors.New("placeholder error") - // GetCPUTime returns the how much CPU time has passed since the start of this process. func GetCPUTime() int64 { var ts unix.Timespec diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go index 3c7a9eb7aff8..d58ed38234d6 100644 --- a/internal/syscall/syscall_nonlinux.go +++ b/internal/syscall/syscall_nonlinux.go @@ -21,15 +21,12 @@ package syscall import ( - "errors" "net" "time" "google.golang.org/grpc/grpclog" ) -var GetTCPUserTimeoutNoopError = errors.New("GetTCPUserTimeout is a no-op on non-linux or appengine environments") - func init() { grpclog.Info("CPU time info is unavailable on non-linux or appengine environment.") } @@ -60,6 +57,7 @@ func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { } // GetTCPUserTimeout is a no-op function under non-linux or appengine environments +// a negative return value indicates the operation is not supported func GetTCPUserTimeout(conn net.Conn) (int, error) { - return 0, GetTCPUserTimeoutNoopError + return -1, nil } diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go index 35b6351d9dbd..b46e097513d9 100644 --- a/internal/transport/transport_test.go +++ b/internal/transport/transport_test.go @@ -2357,11 +2357,11 @@ func TestTCPUserTimeout(t *testing.T) { opt, err := syscall.GetTCPUserTimeout(client.conn) if err != nil { - if err == syscall.GetTCPUserTimeoutNoopError { - t.Skipf("skipping test on unsupported environment: %v", err) - } t.Fatalf("GetTCPUserTimeout error: %v", err) } + if opt < 0 { + t.Skipf("skipping test on unsupported environment") + } if timeoutMS := int(tt.timeout / time.Millisecond); timeoutMS != opt { t.Fatalf("wrong TCP_USER_TIMEOUT set on conn. expected %d. got %d", timeoutMS, opt) From 8581840100a90e23a3b80c71acbff72fc421bac2 Mon Sep 17 00:00:00 2001 From: Michael Raimondi Date: Wed, 17 Oct 2018 21:10:07 -0400 Subject: [PATCH 8/8] use 'setUpWithOptions' so server-side logic is correctly handled --- internal/transport/transport_test.go | 42 +++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go index b46e097513d9..3911a2925d19 100644 --- a/internal/transport/transport_test.go +++ b/internal/transport/transport_test.go @@ -2319,6 +2319,8 @@ func TestHeaderTblSize(t *testing.T) { } } +// TestTCPUserTimeout tests that the TCP_USER_TIMEOUT socket option is set to the +// keepalive timeout, as detailed in proposal A18 func TestTCPUserTimeout(t *testing.T) { tests := []struct { time time.Duration @@ -2334,26 +2336,32 @@ func TestTCPUserTimeout(t *testing.T) { }, } for _, tt := range tests { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Failed to listen. Err: %v", err) - } - defer lis.Close() - // TODO(deklerk): we can `defer cancel()` here after we drop Go 1.6 support. Until then, - // doing a `defer cancel()` could cause the dialer to become broken: - // https://github.com/golang/go/issues/15078, https://github.com/golang/go/issues/15035 - connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) - client, err := newHTTP2Client(connectCtx, context.Background(), TargetInfo{Addr: lis.Addr().String()}, ConnectOptions{ - KeepaliveParams: keepalive.ClientParameters{ - Time: tt.time, - Timeout: tt.timeout, + server, client, cancel := setUpWithOptions( + t, + 0, + &ServerConfig{ + KeepaliveParams: keepalive.ServerParameters{ + Time: tt.timeout, + Timeout: tt.timeout, + }, + }, + normal, + ConnectOptions{ + KeepaliveParams: keepalive.ClientParameters{ + Time: tt.time, + Timeout: tt.timeout, + }, }, - }, func() {}, func(GoAwayReason) {}, func() {}) + ) + defer cancel() + defer server.stop() + defer client.Close() + + stream, err := client.NewStream(context.Background(), &CallHdr{}) if err != nil { - cancel() // Do not cancel in success path. - t.Fatalf("error creating client: %v", err) + t.Fatalf("Client failed to create RPC request: %v", err) } - defer client.Close() + client.closeStream(stream, io.EOF, true, http2.ErrCodeCancel, nil, nil, false) opt, err := syscall.GetTCPUserTimeout(client.conn) if err != nil {