fix(staterpc): install client keepalive on the state store dial
This commit is contained in:
2 files changed
+146
-1
No files matched your search
@@ -4,9 +4,11 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/keepalive"
|
||||
)
|
||||
|
||||
// Dial returns a State Store contract client for target, applying the
|
||||
@@ -34,7 +36,28 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er
|
||||
target,
|
||||
)
|
||||
}
|
||||
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
// Client keepalive. grpc-go's default client keepalive time is
|
||||
// infinity (internal/transport.defaultClientKeepaliveTime), so a
|
||||
// connection that half-opens under an idle client -- peer gone, socket
|
||||
// still ESTABLISHED -- produces no error here. That is load-bearing for
|
||||
// this dial because the control-plane Watch stream is silent by design
|
||||
// while the watched resource's version is unchanged
|
||||
// (internal/app/controlplane's Watch): with neither peer writing,
|
||||
// neither notices, and the caller's watch stays frozen for the life of
|
||||
// the process with no error and no log.
|
||||
//
|
||||
// Time 10s is the soonest a dead peer can be detected: gRPC clamps a
|
||||
// client ping interval up to internal.KeepaliveMinPingTime (10s), and
|
||||
// 10s of connection-level silence costs one HTTP/2 PING, which is far
|
||||
// below the traffic this link already carries (the store service polls
|
||||
// its store every 250ms per watch). Timeout 5s bounds detection at
|
||||
// ~15s: a PING the peer does not ACK within 5s tears the transport
|
||||
// down, turning a half-open connection into a stream error the caller
|
||||
// can act on.
|
||||
grpc.WithKeepaliveParams(clientKeepaliveParams()),
|
||||
}
|
||||
if token != "" {
|
||||
// Both call shapes need the credential: the server's interceptors
|
||||
// guard unary RPCs and the streaming capture reads separately, so a
|
||||
@@ -52,6 +75,20 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er
|
||||
return NewClient(conn), func() { _ = conn.Close() }, nil
|
||||
}
|
||||
|
||||
// clientKeepaliveParams are the client keepalive settings Dial installs; see
|
||||
// Dial's option comment for why each value is what it is. PermitWithoutStream
|
||||
// is deliberately left false (the library default): the pings this dial needs
|
||||
// are covered by an active stream, and pinging a connection with no streams is
|
||||
// what a peer's keepalive enforcement policy records as a ping strike --
|
||||
// against the State Store's default policy a streamless ping draws GOAWAY
|
||||
// too_many_pings and would churn connections that are healthy today.
|
||||
func clientKeepaliveParams() keepalive.ClientParameters {
|
||||
return keepalive.ClientParameters{
|
||||
Time: 10 * time.Second,
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
// TargetIsLoopback reports whether addr's host is a loopback address. Both
|
||||
// literal loopback IPs (127.0.0.1, ::1) and the "localhost" hostname count;
|
||||
// everything else (including the wildcard 0.0.0.0 and DNS names) is treated
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package staterpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
)
|
||||
|
||||
// tcpUserTimeout is TCP_USER_TIMEOUT (Linux, include/uapi/linux/tcp.h). The
|
||||
// stdlib syscall package does not name it; grpc sets it from the client
|
||||
// keepalive timeout on every transport it creates with keepalive enabled
|
||||
// (internal/syscall.SetTCPUserTimeout), which makes it the fastest observable
|
||||
// proof that a connection dialed here carries keepalive at all.
|
||||
const tcpUserTimeout = 18
|
||||
|
||||
// TestDialInstallsClientKeepalive covers the half-open-connection defect: with
|
||||
// no client keepalive the dialed connection carries a keepalive time of
|
||||
// infinity, so a peer that stops delivering bytes (peer gone, socket still
|
||||
// ESTABLISHED) never fails, and a silent control-plane Watch stays frozen for
|
||||
// the life of the process. The assertion is on the dialed socket: grpc only
|
||||
// sets TCP_USER_TIMEOUT from the keepalive timeout when keepalive is enabled,
|
||||
// so a zero value here means the dial installed no keepalive and the defect is
|
||||
// back.
|
||||
func TestDialInstallsClientKeepalive(t *testing.T) {
|
||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
}
|
||||
}()
|
||||
|
||||
// The accepted peer never writes the HTTP/2 preface, so the dialed call
|
||||
// below blocks in its connection attempt; grpc has already applied the
|
||||
// socket options by then, which is all this test reads.
|
||||
dialed := make(chan *net.TCPConn, 1)
|
||||
client, cleanup, err := Dial(listener.Addr().String(), "", grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
|
||||
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tcp, ok := conn.(*net.TCPConn); ok {
|
||||
select {
|
||||
case dialed <- tcp:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return conn, nil
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("dial state store: %v", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
go func() { _, _ = client.ControlPlane().Catalog(ctx, &controlpb.CatalogRequest{}) }()
|
||||
var conn *net.TCPConn
|
||||
select {
|
||||
case conn = <-dialed:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the dial never opened a connection to the peer")
|
||||
}
|
||||
|
||||
// grpc sets the option while creating the transport, after the dialer
|
||||
// returns, so read it once the connect attempt is under way.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var got int
|
||||
for {
|
||||
got = userTimeout(t, conn)
|
||||
if got != 0 || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if got != 5000 {
|
||||
t.Fatalf("dialed connection TCP_USER_TIMEOUT = %d ms, want 5000 (the keepalive timeout): the dial installed no client keepalive, so nothing detects a half-open connection", got)
|
||||
}
|
||||
}
|
||||
|
||||
// userTimeout reads TCP_USER_TIMEOUT in milliseconds off conn's socket.
|
||||
func userTimeout(t *testing.T, conn *net.TCPConn) int {
|
||||
t.Helper()
|
||||
raw, err := conn.SyscallConn()
|
||||
if err != nil {
|
||||
t.Fatalf("raw connection: %v", err)
|
||||
}
|
||||
value := -1
|
||||
if err := raw.Control(func(fd uintptr) {
|
||||
value, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, tcpUserTimeout)
|
||||
}); err != nil {
|
||||
t.Fatalf("read TCP_USER_TIMEOUT: %v", err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
Reference in new issue
Block a user