diff --git a/internal/app/archied/state_store.go b/internal/app/archied/state_store.go index d6a297bda..87808ff1b 100644 --- a/internal/app/archied/state_store.go +++ b/internal/app/archied/state_store.go @@ -414,13 +414,20 @@ func (b *boot) startStateStoreReadiness(ctx context.Context, readyAddr string) e // enforce TLS/mTLS (an operator decision), so it narrows the non-loopback // path to token auth -- the same confinement philosophy as the gateway's // loopback-only --listen rule. +// +// The keepalive enforcement policy is not part of that boundary and is +// installed on both listeners: a loopback State Store serves the same dialers +// as a remote one (staterpc.Dial installs the client keepalive on every target, +// loopback included), and grpc's default policy would answer their idle +// watches with GOAWAY too_many_pings. See staterpc.ServerKeepaliveOption. func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants) (opts []grpc.ServerOption, loopback bool, err error) { loopback, err = staterpc.TargetIsLoopback(listen) if err != nil { return nil, false, err } + keepalive := staterpc.ServerKeepaliveOption() if loopback { - return nil, true, nil + return []grpc.ServerOption{keepalive}, true, nil } if token == "" { return nil, false, fmt.Errorf( @@ -433,6 +440,7 @@ func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants) (op // grant (Update/Transition/InsertEvent on its own task ID only), rather // than the single all-or-nothing token check this replaced. return []grpc.ServerOption{ + keepalive, grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(token)), grpc.ChainStreamInterceptor(grants.StreamInterceptor(token)), }, false, nil diff --git a/internal/app/archied/state_store_keepalive_test.go b/internal/app/archied/state_store_keepalive_test.go new file mode 100644 index 000000000..e86b037dc --- /dev/null +++ b/internal/app/archied/state_store_keepalive_test.go @@ -0,0 +1,149 @@ +package archied + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" +) + +// keepaliveWindow is how long each case below holds one control-plane watch +// open. It has to outlast the point where grpc's default enforcement policy +// tears the link down: the dialer pings every 10s (staterpc's client keepalive +// time, which is also grpc's KeepaliveMinPingTime floor, so a client cannot +// ping sooner), the default EnforcementPolicy.MinTime is 5 minutes, and the +// server sends GOAWAY too_many_pings once the strikes pass 2 (maxPingStrikes). +// Measured: the connection dies 41.0s into the stream. The remaining ~9s are +// there because a loaded machine delays timers, which can only move that +// tear-down later. +const keepaliveWindow = 50 * time.Second + +// TestStateStoreServerOptionsTolerateTheDialersKeepalive holds one watch open +// against two servers: one with grpc's default enforcement policy, one with the +// options the State Store process actually serves with. The default policy is +// the defect -- a server that counts every one of the dialer's pings as a +// strike tears down a connection that carries no traffic for ~40s, and a watch +// is silent by design while the watched resource's version is unchanged +// (internal/app/controlplane's Watch; the dashboard holds the same shape over +// its own connection in internal/webui/api_control_plane.go, writing nothing +// between versions). That case is also what keeps the other one honest: a +// policy that never strikes has nothing to tear down. +func TestStateStoreServerOptionsTolerateTheDialersKeepalive(t *testing.T) { + // The options the standalone State Store serves its local profile with, + // taken from the composition rather than hand-built here. + served, loopback, err := stateStoreServerOpts("127.0.0.1:9090", "", &staterpc.TaskGrants{}) + if err != nil { + t.Fatalf("stateStoreServerOpts: %v", err) + } + if !loopback { + t.Fatal("127.0.0.1 must be reported as loopback") + } + + cases := []struct { + name string + opts []grpc.ServerOption + tornDown bool + }{ + { + // The control: without an enforcement policy coherent with the + // dialer, the same client on the same stream dies mid-window. + name: "grpc's default enforcement policy tears the dialer's watch down", + tornDown: true, + }, + { + name: "the state store's server options keep the dialer's watch up", + opts: served, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + elapsed, recvErr := holdWatch(t, tc.opts, keepaliveWindow) + // The server never writes, so the stream can only end because the + // window ran out or because the connection was torn down under it. + expired := streamEndedByWindow(recvErr) + t.Logf("watch ended after %s with %v", elapsed, recvErr) + if tc.tornDown { + if recvErr == nil || expired { + t.Fatalf("the dialer's watch was still open after %s: the default EnforcementPolicy (MinTime 5m) must strike pings that arrive 10s apart and answer with GOAWAY too_many_pings, so this case is not reaching the strike budget and the case below would prove nothing", keepaliveWindow) + } + if !strings.Contains(recvErr.Error(), "too_many_pings") { + t.Fatalf("the dialer's watch ended after %s with %v: want GOAWAY too_many_pings, not some other connection loss", elapsed, recvErr) + } + return + } + if recvErr == nil || !expired { + t.Fatalf("the dialer's watch ended after %s: %v\nA watch carrying no traffic must survive on the pings staterpc.Dial sends; a server policy that strikes them answers a quiet connection with a dead one at ~40s, which is worse than the keepalive it was added for", elapsed, recvErr) + } + }) + } +} + +// quietWatch is the server half of a control-plane watch that never writes: it +// holds one stream open and sends nothing, the state a watch is in while the +// watched resource's version is unchanged. +type quietWatch struct { + controlpb.UnimplementedControlPlaneServiceServer + started chan struct{} +} + +func (w *quietWatch) Watch(_ *controlpb.WatchRequest, stream controlpb.ControlPlaneService_WatchServer) error { + close(w.started) + <-stream.Context().Done() + return stream.Context().Err() +} + +// holdWatch serves opts over bufconn, dials it with the shipped dialer +// (staterpc.Dial, so the client keepalive is exactly the one the daemon, the +// dashboard and the agent container run), holds one watch stream open for +// window, and reports how long the stream lasted and how it ended. +func holdWatch(t *testing.T, opts []grpc.ServerOption, window time.Duration) (time.Duration, error) { + t.Helper() + listener := bufconn.Listen(1 << 20) + watch := &quietWatch{started: make(chan struct{})} + server := grpc.NewServer(opts...) + controlpb.RegisterControlPlaneServiceServer(server, watch) + served := make(chan struct{}) + go func() { + defer close(served) + _ = server.Serve(listener) + }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + <-served + }) + + client := dialStateStore(t, listener, restartAdminToken) + ctx, cancel := context.WithTimeout(t.Context(), window) + defer cancel() + stream, err := client.ControlPlane().Watch(ctx, &controlpb.WatchRequest{Kind: "any"}) + if err != nil { + t.Fatalf("open watch: %v", err) + } + select { + case <-watch.started: + case <-ctx.Done(): + t.Fatal("the server never started the watch stream") + } + start := time.Now() + // The server never writes, so this returns only when the window ends or the + // connection dies. + _, recvErr := stream.Recv() + return time.Since(start), recvErr +} + +// streamEndedByWindow reports whether err is the window running out rather than +// the connection being torn down under it. +func streamEndedByWindow(err error) bool { + return err != nil && (errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded) +} diff --git a/internal/app/archied/state_store_test.go b/internal/app/archied/state_store_test.go index fede02cf5..be8e9d9b6 100644 --- a/internal/app/archied/state_store_test.go +++ b/internal/app/archied/state_store_test.go @@ -25,8 +25,8 @@ func TestStateStoreServerOptsLoopbackIsInsecure(t *testing.T) { if !loopback { t.Fatal("loopback listener should report loopback") } - if len(opts) != 0 { - t.Fatalf("loopback listener should have no server opts, got %d", len(opts)) + if len(opts) != 1 { + t.Fatalf("loopback listener should install only the keepalive enforcement policy (no token interceptors), got %d server opts", len(opts)) } } @@ -41,8 +41,8 @@ func TestStateStoreServerOptsNonLoopbackRequiresToken(t *testing.T) { if loopback { t.Fatal("non-loopback listener should not report loopback") } - if len(opts) != 2 { - t.Fatalf("non-loopback listener should install two server options (the unary and stream task-grant interceptors), got %d", len(opts)) + if len(opts) != 3 { + t.Fatalf("non-loopback listener should install three server options (the keepalive enforcement policy and the unary and stream task-grant interceptors), got %d", len(opts)) } } diff --git a/internal/infrastructure/staterpc/dial.go b/internal/infrastructure/staterpc/dial.go index befda9d90..9290cdfb2 100644 --- a/internal/infrastructure/staterpc/dial.go +++ b/internal/infrastructure/staterpc/dial.go @@ -89,6 +89,53 @@ func clientKeepaliveParams() keepalive.ClientParameters { } } +// serverKeepaliveMinTime is the server half of the same agreement: the +// shortest gap between two pings a State Store server accepts before it counts +// a strike. It is half the client's ping interval rather than a copy of it; +// ServerKeepaliveOption carries the arithmetic. +const serverKeepaliveMinTime = 5 * time.Second + +// ServerKeepaliveOption returns the server half of the keepalive agreement +// Dial's clients enter, for the composition that serves the State Store +// contract to install on its grpc.Server (internal/app/archied's +// stateStoreServerOpts installs it on every listener). +// +// The two sides have to agree on one number. A ping arriving less than +// EnforcementPolicy.MinTime after the previous ping is a strike, and the server +// answers the third strike with GOAWAY too_many_pings (maxPingStrikes is 2; a +// write on that transport is what resets the counter). grpc's default policy +// MinTime is 5 minutes while the client above pings every 10 seconds, so a +// connection that carries no write for ~40s -- which is what a watch is while +// the watched resource's version is unchanged -- is torn down where before the +// client keepalive existed it stayed up. A process that writes on the same +// transport every 30s (the apply-status re-stamp, applystatus.RestampInterval) +// resets the counter often enough to hide this; an idle connection is what it +// kills, and a keepalive that disconnects is worse than no keepalive. +// +// MinTime is 5s, half the client's ping interval. The client's interval is the +// larger of its Time and grpc's KeepaliveMinPingTime clamp -- 10s, so no caller +// can ask for faster pings -- and its keepalive loop arms its timer Time ahead +// and re-arms it only later, on read activity, so consecutive pings are never +// closer than ~10s apart. A 5s floor thus separates every ping from a strike by +// a factor of two, worth more than Go's timer granularity and any scheduling +// delay, which can only push pings further apart. A MinTime of 10s would sit +// exactly on the boundary the clamp already enforces, where a ping a +// microsecond early is a strike; 5 minutes is the defect itself. +// +// The floor is not a blanket exemption: a ping arriving less than 5s after the +// previous one, and any ping on a connection with no stream, still strikes. +// PermitWithoutStream stays false, matching the client, which never pings a +// connection with no streams: every client this server has is a staterpc.Dial +// -- the daemon's control-plane client, the archie-ui dashboard, archie- +// messaging and the archie-agent container -- so permitting streamless pings +// would only relax a setting none of them exercises. +func ServerKeepaliveOption() grpc.ServerOption { + return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: serverKeepaliveMinTime, + PermitWithoutStream: false, + }) +} + // TargetIsLoopback reports whether addr's host is a loopback address. Both // literal loopback IPs (127.0.0.1, ::1) and the "localhost" hostname count; // everything else (including the wildcard 0.0.0.0 and DNS names) is treated