fix(staterpc): install client keepalive on the state store dial

This commit is contained in:
sam committed 2026-09-22 01:57:56 +10:00
1 parent 28f7d2a0f7
commit 6c8d531361
2 files changed
+146 -1

No files matched your search

+38 -1
View File
@@ -4,9 +4,11 @@ import (
"fmt"
"net"
"strings"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive"
)
// Dial returns a State Store contract client for target, applying the
@@ -34,7 +36,28 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er
target,
)
}
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
// Client keepalive. grpc-go's default client keepalive time is
// infinity (internal/transport.defaultClientKeepaliveTime), so a
// connection that half-opens under an idle client -- peer gone, socket
// still ESTABLISHED -- produces no error here. That is load-bearing for
// this dial because the control-plane Watch stream is silent by design
// while the watched resource's version is unchanged
// (internal/app/controlplane's Watch): with neither peer writing,
// neither notices, and the caller's watch stays frozen for the life of
// the process with no error and no log.
//
// Time 10s is the soonest a dead peer can be detected: gRPC clamps a
// client ping interval up to internal.KeepaliveMinPingTime (10s), and
// 10s of connection-level silence costs one HTTP/2 PING, which is far
// below the traffic this link already carries (the store service polls
// its store every 250ms per watch). Timeout 5s bounds detection at
// ~15s: a PING the peer does not ACK within 5s tears the transport
// down, turning a half-open connection into a stream error the caller
// can act on.
grpc.WithKeepaliveParams(clientKeepaliveParams()),
}
if token != "" {
// Both call shapes need the credential: the server's interceptors
// guard unary RPCs and the streaming capture reads separately, so a
@@ -52,6 +75,20 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er
return NewClient(conn), func() { _ = conn.Close() }, nil
}
// clientKeepaliveParams are the client keepalive settings Dial installs; see
// Dial's option comment for why each value is what it is. PermitWithoutStream
// is deliberately left false (the library default): the pings this dial needs
// are covered by an active stream, and pinging a connection with no streams is
// what a peer's keepalive enforcement policy records as a ping strike --
// against the State Store's default policy a streamless ping draws GOAWAY
// too_many_pings and would churn connections that are healthy today.
func clientKeepaliveParams() keepalive.ClientParameters {
return keepalive.ClientParameters{
Time: 10 * time.Second,
Timeout: 5 * time.Second,
}
}
// TargetIsLoopback reports whether addr's host is a loopback address. Both
// literal loopback IPs (127.0.0.1, ::1) and the "localhost" hostname count;
// everything else (including the wildcard 0.0.0.0 and DNS names) is treated
@@ -0,0 +1,108 @@
package staterpc
import (
"context"
"net"
"syscall"
"testing"
"time"
"google.golang.org/grpc"
controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
)
// tcpUserTimeout is TCP_USER_TIMEOUT (Linux, include/uapi/linux/tcp.h). The
// stdlib syscall package does not name it; grpc sets it from the client
// keepalive timeout on every transport it creates with keepalive enabled
// (internal/syscall.SetTCPUserTimeout), which makes it the fastest observable
// proof that a connection dialed here carries keepalive at all.
const tcpUserTimeout = 18
// TestDialInstallsClientKeepalive covers the half-open-connection defect: with
// no client keepalive the dialed connection carries a keepalive time of
// infinity, so a peer that stops delivering bytes (peer gone, socket still
// ESTABLISHED) never fails, and a silent control-plane Watch stays frozen for
// the life of the process. The assertion is on the dialed socket: grpc only
// sets TCP_USER_TIMEOUT from the keepalive timeout when keepalive is enabled,
// so a zero value here means the dial installed no keepalive and the defect is
// back.
func TestDialInstallsClientKeepalive(t *testing.T) {
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
t.Cleanup(func() { _ = conn.Close() })
}
}()
// The accepted peer never writes the HTTP/2 preface, so the dialed call
// below blocks in its connection attempt; grpc has already applied the
// socket options by then, which is all this test reads.
dialed := make(chan *net.TCPConn, 1)
client, cleanup, err := Dial(listener.Addr().String(), "", grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr)
if err != nil {
return nil, err
}
if tcp, ok := conn.(*net.TCPConn); ok {
select {
case dialed <- tcp:
default:
}
}
return conn, nil
}))
if err != nil {
t.Fatalf("dial state store: %v", err)
}
defer cleanup()
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
go func() { _, _ = client.ControlPlane().Catalog(ctx, &controlpb.CatalogRequest{}) }()
var conn *net.TCPConn
select {
case conn = <-dialed:
case <-ctx.Done():
t.Fatal("the dial never opened a connection to the peer")
}
// grpc sets the option while creating the transport, after the dialer
// returns, so read it once the connect attempt is under way.
deadline := time.Now().Add(2 * time.Second)
var got int
for {
got = userTimeout(t, conn)
if got != 0 || time.Now().After(deadline) {
break
}
time.Sleep(10 * time.Millisecond)
}
if got != 5000 {
t.Fatalf("dialed connection TCP_USER_TIMEOUT = %d ms, want 5000 (the keepalive timeout): the dial installed no client keepalive, so nothing detects a half-open connection", got)
}
}
// userTimeout reads TCP_USER_TIMEOUT in milliseconds off conn's socket.
func userTimeout(t *testing.T, conn *net.TCPConn) int {
t.Helper()
raw, err := conn.SyscallConn()
if err != nil {
t.Fatalf("raw connection: %v", err)
}
value := -1
if err := raw.Control(func(fd uintptr) {
value, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, tcpUserTimeout)
}); err != nil {
t.Fatalf("read TCP_USER_TIMEOUT: %v", err)
}
return value
}