diff --git a/internal/infrastructure/staterpc/dial.go b/internal/infrastructure/staterpc/dial.go index a3717671a..befda9d90 100644 --- a/internal/infrastructure/staterpc/dial.go +++ b/internal/infrastructure/staterpc/dial.go @@ -4,9 +4,11 @@ import ( "fmt" "net" "strings" + "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" ) // Dial returns a State Store contract client for target, applying the @@ -34,7 +36,28 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er target, ) } - opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + // Client keepalive. grpc-go's default client keepalive time is + // infinity (internal/transport.defaultClientKeepaliveTime), so a + // connection that half-opens under an idle client -- peer gone, socket + // still ESTABLISHED -- produces no error here. That is load-bearing for + // this dial because the control-plane Watch stream is silent by design + // while the watched resource's version is unchanged + // (internal/app/controlplane's Watch): with neither peer writing, + // neither notices, and the caller's watch stays frozen for the life of + // the process with no error and no log. + // + // Time 10s is the soonest a dead peer can be detected: gRPC clamps a + // client ping interval up to internal.KeepaliveMinPingTime (10s), and + // 10s of connection-level silence costs one HTTP/2 PING, which is far + // below the traffic this link already carries (the store service polls + // its store every 250ms per watch). Timeout 5s bounds detection at + // ~15s: a PING the peer does not ACK within 5s tears the transport + // down, turning a half-open connection into a stream error the caller + // can act on. + grpc.WithKeepaliveParams(clientKeepaliveParams()), + } if token != "" { // Both call shapes need the credential: the server's interceptors // guard unary RPCs and the streaming capture reads separately, so a @@ -52,6 +75,20 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er return NewClient(conn), func() { _ = conn.Close() }, nil } +// clientKeepaliveParams are the client keepalive settings Dial installs; see +// Dial's option comment for why each value is what it is. PermitWithoutStream +// is deliberately left false (the library default): the pings this dial needs +// are covered by an active stream, and pinging a connection with no streams is +// what a peer's keepalive enforcement policy records as a ping strike -- +// against the State Store's default policy a streamless ping draws GOAWAY +// too_many_pings and would churn connections that are healthy today. +func clientKeepaliveParams() keepalive.ClientParameters { + return keepalive.ClientParameters{ + Time: 10 * time.Second, + Timeout: 5 * time.Second, + } +} + // TargetIsLoopback reports whether addr's host is a loopback address. Both // literal loopback IPs (127.0.0.1, ::1) and the "localhost" hostname count; // everything else (including the wildcard 0.0.0.0 and DNS names) is treated diff --git a/internal/infrastructure/staterpc/dial_keepalive_test.go b/internal/infrastructure/staterpc/dial_keepalive_test.go new file mode 100644 index 000000000..d252cdce2 --- /dev/null +++ b/internal/infrastructure/staterpc/dial_keepalive_test.go @@ -0,0 +1,108 @@ +package staterpc + +import ( + "context" + "net" + "syscall" + "testing" + "time" + + "google.golang.org/grpc" + + controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" +) + +// tcpUserTimeout is TCP_USER_TIMEOUT (Linux, include/uapi/linux/tcp.h). The +// stdlib syscall package does not name it; grpc sets it from the client +// keepalive timeout on every transport it creates with keepalive enabled +// (internal/syscall.SetTCPUserTimeout), which makes it the fastest observable +// proof that a connection dialed here carries keepalive at all. +const tcpUserTimeout = 18 + +// TestDialInstallsClientKeepalive covers the half-open-connection defect: with +// no client keepalive the dialed connection carries a keepalive time of +// infinity, so a peer that stops delivering bytes (peer gone, socket still +// ESTABLISHED) never fails, and a silent control-plane Watch stays frozen for +// the life of the process. The assertion is on the dialed socket: grpc only +// sets TCP_USER_TIMEOUT from the keepalive timeout when keepalive is enabled, +// so a zero value here means the dial installed no keepalive and the defect is +// back. +func TestDialInstallsClientKeepalive(t *testing.T) { + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = conn.Close() }) + } + }() + + // The accepted peer never writes the HTTP/2 preface, so the dialed call + // below blocks in its connection attempt; grpc has already applied the + // socket options by then, which is all this test reads. + dialed := make(chan *net.TCPConn, 1) + client, cleanup, err := Dial(listener.Addr().String(), "", grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + if tcp, ok := conn.(*net.TCPConn); ok { + select { + case dialed <- tcp: + default: + } + } + return conn, nil + })) + if err != nil { + t.Fatalf("dial state store: %v", err) + } + defer cleanup() + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + go func() { _, _ = client.ControlPlane().Catalog(ctx, &controlpb.CatalogRequest{}) }() + var conn *net.TCPConn + select { + case conn = <-dialed: + case <-ctx.Done(): + t.Fatal("the dial never opened a connection to the peer") + } + + // grpc sets the option while creating the transport, after the dialer + // returns, so read it once the connect attempt is under way. + deadline := time.Now().Add(2 * time.Second) + var got int + for { + got = userTimeout(t, conn) + if got != 0 || time.Now().After(deadline) { + break + } + time.Sleep(10 * time.Millisecond) + } + if got != 5000 { + t.Fatalf("dialed connection TCP_USER_TIMEOUT = %d ms, want 5000 (the keepalive timeout): the dial installed no client keepalive, so nothing detects a half-open connection", got) + } +} + +// userTimeout reads TCP_USER_TIMEOUT in milliseconds off conn's socket. +func userTimeout(t *testing.T, conn *net.TCPConn) int { + t.Helper() + raw, err := conn.SyscallConn() + if err != nil { + t.Fatalf("raw connection: %v", err) + } + value := -1 + if err := raw.Control(func(fd uintptr) { + value, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, tcpUserTimeout) + }); err != nil { + t.Fatalf("read TCP_USER_TIMEOUT: %v", err) + } + return value +}