diff --git a/internal/app/archied/state_store_keepalive_test.go b/internal/app/archied/state_store_keepalive_test.go index e86b037dc..980d68fa6 100644 --- a/internal/app/archied/state_store_keepalive_test.go +++ b/internal/app/archied/state_store_keepalive_test.go @@ -16,6 +16,69 @@ import ( "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" ) +// keepaliveDialerMargin is the headroom the enforcement floor keeps under the +// dialer's ping interval: the floor is half that interval, so a ping lands a +// factor of two clear of a strike rather than on the boundary. +const keepaliveDialerMargin = 2 + +// TestStateStoreServerOptsHoldTheDialersKeepaliveMargin is the cheap half of +// the keepalive guard, and the only half that runs when someone changes the +// policy: it asserts in microseconds what the stream case below proves over the +// control's tear-down time (measured: 41.0s), so a server half that stops +// tolerating the dialer's pings fails immediately instead of at the end of a +// 50s window. +// +// A grpc.ServerOption is a closure over unexported grpc state, so neither the +// option stateStoreServerOpts returns nor the one ServerKeepaliveOption builds +// can be read back once built. What is asserted is therefore the two facts the +// chain is made of: the shape of the option list (a loopback listener installs +// exactly one, the enforcement policy, and nothing else) and the value that +// option is built from, staterpc.ServerKeepalivePolicy. Both numbers in the +// agreement are read from staterpc, the package that dials the client and +// serves the option, so no assertion here restates a value one of the halves +// owns. +func TestStateStoreServerOptsHoldTheDialersKeepaliveMargin(t *testing.T) { + t.Parallel() + + opts, loopback, err := stateStoreServerOpts("127.0.0.1:9090", "", &staterpc.TaskGrants{}) + if err != nil { + t.Fatalf("stateStoreServerOpts: %v", err) + } + if !loopback { + t.Fatal("127.0.0.1 must be reported as loopback") + } + // The token interceptors belong to the non-loopback branch, so exactly one + // option here is the enforcement policy and nothing else: fewer means the + // server serves grpc's default policy (MinTime 5m), which answers the + // dialer's pings with GOAWAY too_many_pings and tears a quiet watch down at + // ~41s. + if len(opts) != 1 { + t.Fatalf("the loopback listener installs %d server options, want 1 (the keepalive enforcement policy)", len(opts)) + } + + floor := staterpc.ServerKeepalivePolicy() + dialer := staterpc.ClientKeepaliveParams() + + if floor.MinTime != 5*time.Second { + t.Fatalf("the enforcement floor is %s, want 5s: grpc's default 5m is the defect itself, and 10s would sit on the dialer's own KeepaliveMinPingTime clamp, where a ping a microsecond early is a strike", floor.MinTime) + } + if floor.PermitWithoutStream { + t.Fatal("PermitWithoutStream = true: the dialer never pings a connection with no streams, so permitting them only relaxes a setting nothing exercises -- and a streamless ping from any other client would draw strikes against the dialer's traffic") + } + + // The relationship, not the literal, is what makes the pair correct: the + // dialer's keepalive loop arms its timer one Time ahead and re-arms only + // later (on read activity), so consecutive pings are never closer than Time + // apart. A floor below that interval is never reached; a floor equal to it + // is reached by any ping that arrives early. + if floor.MinTime >= dialer.Time { + t.Fatalf("the enforcement floor %s is not below the dialer's %s ping interval: every ping would land on the strike boundary", floor.MinTime, dialer.Time) + } + if floor.MinTime*keepaliveDialerMargin > dialer.Time { + t.Fatalf("the enforcement floor %s must be at most %s (a %dx margin under the dialer's %s ping interval): the margin is what a loaded machine's timer delay eats before a ping draws a strike", floor.MinTime, dialer.Time/keepaliveDialerMargin, keepaliveDialerMargin, dialer.Time) + } +} + // keepaliveWindow is how long each case below holds one control-plane watch // open. It has to outlast the point where grpc's default enforcement policy // tears the link down: the dialer pings every 10s (staterpc's client keepalive @@ -37,6 +100,13 @@ const keepaliveWindow = 50 * time.Second // its own connection in internal/webui/api_control_plane.go, writing nothing // between versions). That case is also what keeps the other one honest: a // policy that never strikes has nothing to tear down. +// +// The pair's cases run concurrently (t.Parallel) and share no state: each has +// its own bufconn listener, grpc.Server, dialed client and deadline, and each +// asserts only on its own stream, so neither one's timing depends on the +// other's scheduling. TestStateStoreServerOptsHoldTheDialersKeepaliveMargin +// above pins the same agreement in microseconds; this case is the end-to-end +// proof, which is the part a value assertion cannot give. func TestStateStoreServerOptionsTolerateTheDialersKeepalive(t *testing.T) { // The options the standalone State Store serves its local profile with, // taken from the composition rather than hand-built here. diff --git a/internal/infrastructure/staterpc/dial.go b/internal/infrastructure/staterpc/dial.go index 9290cdfb2..ff5b96266 100644 --- a/internal/infrastructure/staterpc/dial.go +++ b/internal/infrastructure/staterpc/dial.go @@ -56,7 +56,7 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er // ~15s: a PING the peer does not ACK within 5s tears the transport // down, turning a half-open connection into a stream error the caller // can act on. - grpc.WithKeepaliveParams(clientKeepaliveParams()), + grpc.WithKeepaliveParams(ClientKeepaliveParams()), } if token != "" { // Both call shapes need the credential: the server's interceptors @@ -75,14 +75,19 @@ func Dial(target, token string, options ...grpc.DialOption) (*Client, func(), er return NewClient(conn), func() { _ = conn.Close() }, nil } -// clientKeepaliveParams are the client keepalive settings Dial installs; see +// ClientKeepaliveParams are the client keepalive settings Dial installs; see // Dial's option comment for why each value is what it is. PermitWithoutStream // is deliberately left false (the library default): the pings this dial needs // are covered by an active stream, and pinging a connection with no streams is // what a peer's keepalive enforcement policy records as a ping strike -- // against the State Store's default policy a streamless ping draws GOAWAY // too_many_pings and would churn connections that are healthy today. -func clientKeepaliveParams() keepalive.ClientParameters { +// +// It is the dialer's half of the keepalive agreement with +// ServerKeepalivePolicy, so it is exported: the side that installs a server +// option reads the interval this dialer actually pings at from here rather +// than restating it. +func ClientKeepaliveParams() keepalive.ClientParameters { return keepalive.ClientParameters{ Time: 10 * time.Second, Timeout: 5 * time.Second, @@ -130,10 +135,20 @@ const serverKeepaliveMinTime = 5 * time.Second // messaging and the archie-agent container -- so permitting streamless pings // would only relax a setting none of them exercises. func ServerKeepaliveOption() grpc.ServerOption { - return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + return grpc.KeepaliveEnforcementPolicy(ServerKeepalivePolicy()) +} + +// ServerKeepalivePolicy returns the policy ServerKeepaliveOption installs as a +// value. A grpc.ServerOption is a closure over unexported grpc state with no +// accessor, so the option cannot be read back once built; this value is +// therefore the handle the agreement is held to -- a caller installing the +// option can assert the policy it is built from, and ClientKeepaliveParams +// gives the interval that policy must stay below. +func ServerKeepalivePolicy() keepalive.EnforcementPolicy { + return keepalive.EnforcementPolicy{ MinTime: serverKeepaliveMinTime, PermitWithoutStream: false, - }) + } } // TargetIsLoopback reports whether addr's host is a loopback address. Both