fix(staterpc): make the state store's server policy accept the dialer's pings

The dialer's keepalive pings a server running grpc's default enforcement policy (MinTime 5m), which counts every ping arriving less than five minutes after the previous one as a strike and answers the third with GOAWAY too_many_pings. The dialer pings every 10s, so a connection that carries no write for ~40s -- a watch while the watched resource's version is unchanged -- was torn down, where before the client keepalive it stayed up.

ServerKeepaliveOption returns MinTime 5s, half the client's ping interval, so every ping lands 5s clear of a strike; stateStoreServerOpts installs it on the loopback and token listeners alike.
This commit is contained in:
sam committed 2026-09-22 01:57:56 +10:00
1 parent 6c8d531361
commit 8c3136eb2b
4 files changed
+209 -5

No files matched your search

+9 -1
View File
@@ -414,13 +414,20 @@ func (b *boot) startStateStoreReadiness(ctx context.Context, readyAddr string) e
// enforce TLS/mTLS (an operator decision), so it narrows the non-loopback
// path to token auth -- the same confinement philosophy as the gateway's
// loopback-only --listen rule.
//
// The keepalive enforcement policy is not part of that boundary and is
// installed on both listeners: a loopback State Store serves the same dialers
// as a remote one (staterpc.Dial installs the client keepalive on every target,
// loopback included), and grpc's default policy would answer their idle
// watches with GOAWAY too_many_pings. See staterpc.ServerKeepaliveOption.
func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants) (opts []grpc.ServerOption, loopback bool, err error) {
loopback, err = staterpc.TargetIsLoopback(listen)
if err != nil {
return nil, false, err
}
keepalive := staterpc.ServerKeepaliveOption()
if loopback {
return nil, true, nil
return []grpc.ServerOption{keepalive}, true, nil
}
if token == "" {
return nil, false, fmt.Errorf(
@@ -433,6 +440,7 @@ func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants) (op
// grant (Update/Transition/InsertEvent on its own task ID only), rather
// than the single all-or-nothing token check this replaced.
return []grpc.ServerOption{
keepalive,
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(token)),
grpc.ChainStreamInterceptor(grants.StreamInterceptor(token)),
}, false, nil
@@ -0,0 +1,149 @@
package archied
import (
"context"
"errors"
"strings"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/grpc/test/bufconn"
controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
)
// keepaliveWindow is how long each case below holds one control-plane watch
// open. It has to outlast the point where grpc's default enforcement policy
// tears the link down: the dialer pings every 10s (staterpc's client keepalive
// time, which is also grpc's KeepaliveMinPingTime floor, so a client cannot
// ping sooner), the default EnforcementPolicy.MinTime is 5 minutes, and the
// server sends GOAWAY too_many_pings once the strikes pass 2 (maxPingStrikes).
// Measured: the connection dies 41.0s into the stream. The remaining ~9s are
// there because a loaded machine delays timers, which can only move that
// tear-down later.
const keepaliveWindow = 50 * time.Second
// TestStateStoreServerOptionsTolerateTheDialersKeepalive holds one watch open
// against two servers: one with grpc's default enforcement policy, one with the
// options the State Store process actually serves with. The default policy is
// the defect -- a server that counts every one of the dialer's pings as a
// strike tears down a connection that carries no traffic for ~40s, and a watch
// is silent by design while the watched resource's version is unchanged
// (internal/app/controlplane's Watch; the dashboard holds the same shape over
// its own connection in internal/webui/api_control_plane.go, writing nothing
// between versions). That case is also what keeps the other one honest: a
// policy that never strikes has nothing to tear down.
func TestStateStoreServerOptionsTolerateTheDialersKeepalive(t *testing.T) {
// The options the standalone State Store serves its local profile with,
// taken from the composition rather than hand-built here.
served, loopback, err := stateStoreServerOpts("127.0.0.1:9090", "", &staterpc.TaskGrants{})
if err != nil {
t.Fatalf("stateStoreServerOpts: %v", err)
}
if !loopback {
t.Fatal("127.0.0.1 must be reported as loopback")
}
cases := []struct {
name string
opts []grpc.ServerOption
tornDown bool
}{
{
// The control: without an enforcement policy coherent with the
// dialer, the same client on the same stream dies mid-window.
name: "grpc's default enforcement policy tears the dialer's watch down",
tornDown: true,
},
{
name: "the state store's server options keep the dialer's watch up",
opts: served,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
elapsed, recvErr := holdWatch(t, tc.opts, keepaliveWindow)
// The server never writes, so the stream can only end because the
// window ran out or because the connection was torn down under it.
expired := streamEndedByWindow(recvErr)
t.Logf("watch ended after %s with %v", elapsed, recvErr)
if tc.tornDown {
if recvErr == nil || expired {
t.Fatalf("the dialer's watch was still open after %s: the default EnforcementPolicy (MinTime 5m) must strike pings that arrive 10s apart and answer with GOAWAY too_many_pings, so this case is not reaching the strike budget and the case below would prove nothing", keepaliveWindow)
}
if !strings.Contains(recvErr.Error(), "too_many_pings") {
t.Fatalf("the dialer's watch ended after %s with %v: want GOAWAY too_many_pings, not some other connection loss", elapsed, recvErr)
}
return
}
if recvErr == nil || !expired {
t.Fatalf("the dialer's watch ended after %s: %v\nA watch carrying no traffic must survive on the pings staterpc.Dial sends; a server policy that strikes them answers a quiet connection with a dead one at ~40s, which is worse than the keepalive it was added for", elapsed, recvErr)
}
})
}
}
// quietWatch is the server half of a control-plane watch that never writes: it
// holds one stream open and sends nothing, the state a watch is in while the
// watched resource's version is unchanged.
type quietWatch struct {
controlpb.UnimplementedControlPlaneServiceServer
started chan struct{}
}
func (w *quietWatch) Watch(_ *controlpb.WatchRequest, stream controlpb.ControlPlaneService_WatchServer) error {
close(w.started)
<-stream.Context().Done()
return stream.Context().Err()
}
// holdWatch serves opts over bufconn, dials it with the shipped dialer
// (staterpc.Dial, so the client keepalive is exactly the one the daemon, the
// dashboard and the agent container run), holds one watch stream open for
// window, and reports how long the stream lasted and how it ended.
func holdWatch(t *testing.T, opts []grpc.ServerOption, window time.Duration) (time.Duration, error) {
t.Helper()
listener := bufconn.Listen(1 << 20)
watch := &quietWatch{started: make(chan struct{})}
server := grpc.NewServer(opts...)
controlpb.RegisterControlPlaneServiceServer(server, watch)
served := make(chan struct{})
go func() {
defer close(served)
_ = server.Serve(listener)
}()
t.Cleanup(func() {
server.Stop()
_ = listener.Close()
<-served
})
client := dialStateStore(t, listener, restartAdminToken)
ctx, cancel := context.WithTimeout(t.Context(), window)
defer cancel()
stream, err := client.ControlPlane().Watch(ctx, &controlpb.WatchRequest{Kind: "any"})
if err != nil {
t.Fatalf("open watch: %v", err)
}
select {
case <-watch.started:
case <-ctx.Done():
t.Fatal("the server never started the watch stream")
}
start := time.Now()
// The server never writes, so this returns only when the window ends or the
// connection dies.
_, recvErr := stream.Recv()
return time.Since(start), recvErr
}
// streamEndedByWindow reports whether err is the window running out rather than
// the connection being torn down under it.
func streamEndedByWindow(err error) bool {
return err != nil && (errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded)
}
+4 -4
View File
@@ -25,8 +25,8 @@ func TestStateStoreServerOptsLoopbackIsInsecure(t *testing.T) {
if !loopback {
t.Fatal("loopback listener should report loopback")
}
if len(opts) != 0 {
t.Fatalf("loopback listener should have no server opts, got %d", len(opts))
if len(opts) != 1 {
t.Fatalf("loopback listener should install only the keepalive enforcement policy (no token interceptors), got %d server opts", len(opts))
}
}
@@ -41,8 +41,8 @@ func TestStateStoreServerOptsNonLoopbackRequiresToken(t *testing.T) {
if loopback {
t.Fatal("non-loopback listener should not report loopback")
}
if len(opts) != 2 {
t.Fatalf("non-loopback listener should install two server options (the unary and stream task-grant interceptors), got %d", len(opts))
if len(opts) != 3 {
t.Fatalf("non-loopback listener should install three server options (the keepalive enforcement policy and the unary and stream task-grant interceptors), got %d", len(opts))
}
}
+47
View File
@@ -89,6 +89,53 @@ func clientKeepaliveParams() keepalive.ClientParameters {
}
}
// serverKeepaliveMinTime is the server half of the same agreement: the
// shortest gap between two pings a State Store server accepts before it counts
// a strike. It is half the client's ping interval rather than a copy of it;
// ServerKeepaliveOption carries the arithmetic.
const serverKeepaliveMinTime = 5 * time.Second
// ServerKeepaliveOption returns the server half of the keepalive agreement
// Dial's clients enter, for the composition that serves the State Store
// contract to install on its grpc.Server (internal/app/archied's
// stateStoreServerOpts installs it on every listener).
//
// The two sides have to agree on one number. A ping arriving less than
// EnforcementPolicy.MinTime after the previous ping is a strike, and the server
// answers the third strike with GOAWAY too_many_pings (maxPingStrikes is 2; a
// write on that transport is what resets the counter). grpc's default policy
// MinTime is 5 minutes while the client above pings every 10 seconds, so a
// connection that carries no write for ~40s -- which is what a watch is while
// the watched resource's version is unchanged -- is torn down where before the
// client keepalive existed it stayed up. A process that writes on the same
// transport every 30s (the apply-status re-stamp, applystatus.RestampInterval)
// resets the counter often enough to hide this; an idle connection is what it
// kills, and a keepalive that disconnects is worse than no keepalive.
//
// MinTime is 5s, half the client's ping interval. The client's interval is the
// larger of its Time and grpc's KeepaliveMinPingTime clamp -- 10s, so no caller
// can ask for faster pings -- and its keepalive loop arms its timer Time ahead
// and re-arms it only later, on read activity, so consecutive pings are never
// closer than ~10s apart. A 5s floor thus separates every ping from a strike by
// a factor of two, worth more than Go's timer granularity and any scheduling
// delay, which can only push pings further apart. A MinTime of 10s would sit
// exactly on the boundary the clamp already enforces, where a ping a
// microsecond early is a strike; 5 minutes is the defect itself.
//
// The floor is not a blanket exemption: a ping arriving less than 5s after the
// previous one, and any ping on a connection with no stream, still strikes.
// PermitWithoutStream stays false, matching the client, which never pings a
// connection with no streams: every client this server has is a staterpc.Dial
// -- the daemon's control-plane client, the archie-ui dashboard, archie-
// messaging and the archie-agent container -- so permitting streamless pings
// would only relax a setting none of them exercises.
func ServerKeepaliveOption() grpc.ServerOption {
return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
MinTime: serverKeepaliveMinTime,
PermitWithoutStream: false,
})
}
// TargetIsLoopback reports whether addr's host is a loopback address. Both
// literal loopback IPs (127.0.0.1, ::1) and the "localhost" hostname count;
// everything else (including the wildcard 0.0.0.0 and DNS names) is treated