fix(staterpc): make the state store's server policy accept the dialer's pings
The dialer's keepalive pings a server running grpc's default enforcement policy (MinTime 5m), which counts every ping arriving less than five minutes after the previous one as a strike and answers the third with GOAWAY too_many_pings. The dialer pings every 10s, so a connection that carries no write for ~40s -- a watch while the watched resource's version is unchanged -- was torn down, where before the client keepalive it stayed up. ServerKeepaliveOption returns MinTime 5s, half the client's ping interval, so every ping lands 5s clear of a strike; stateStoreServerOpts installs it on the loopback and token listeners alike.
This commit is contained in:
4 files changed
+209
-5
No files matched your search
@@ -414,13 +414,20 @@ func (b *boot) startStateStoreReadiness(ctx context.Context, readyAddr string) e
|
||||
// enforce TLS/mTLS (an operator decision), so it narrows the non-loopback
|
||||
// path to token auth -- the same confinement philosophy as the gateway's
|
||||
// loopback-only --listen rule.
|
||||
//
|
||||
// The keepalive enforcement policy is not part of that boundary and is
|
||||
// installed on both listeners: a loopback State Store serves the same dialers
|
||||
// as a remote one (staterpc.Dial installs the client keepalive on every target,
|
||||
// loopback included), and grpc's default policy would answer their idle
|
||||
// watches with GOAWAY too_many_pings. See staterpc.ServerKeepaliveOption.
|
||||
func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants) (opts []grpc.ServerOption, loopback bool, err error) {
|
||||
loopback, err = staterpc.TargetIsLoopback(listen)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
keepalive := staterpc.ServerKeepaliveOption()
|
||||
if loopback {
|
||||
return nil, true, nil
|
||||
return []grpc.ServerOption{keepalive}, true, nil
|
||||
}
|
||||
if token == "" {
|
||||
return nil, false, fmt.Errorf(
|
||||
@@ -433,6 +440,7 @@ func stateStoreServerOpts(listen, token string, grants *staterpc.TaskGrants) (op
|
||||
// grant (Update/Transition/InsertEvent on its own task ID only), rather
|
||||
// than the single all-or-nothing token check this replaced.
|
||||
return []grpc.ServerOption{
|
||||
keepalive,
|
||||
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(token)),
|
||||
grpc.ChainStreamInterceptor(grants.StreamInterceptor(token)),
|
||||
}, false, nil
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/grpc/test/bufconn"
|
||||
|
||||
controlpb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
)
|
||||
|
||||
// keepaliveWindow is how long each case below holds one control-plane watch
|
||||
// open. It has to outlast the point where grpc's default enforcement policy
|
||||
// tears the link down: the dialer pings every 10s (staterpc's client keepalive
|
||||
// time, which is also grpc's KeepaliveMinPingTime floor, so a client cannot
|
||||
// ping sooner), the default EnforcementPolicy.MinTime is 5 minutes, and the
|
||||
// server sends GOAWAY too_many_pings once the strikes pass 2 (maxPingStrikes).
|
||||
// Measured: the connection dies 41.0s into the stream. The remaining ~9s are
|
||||
// there because a loaded machine delays timers, which can only move that
|
||||
// tear-down later.
|
||||
const keepaliveWindow = 50 * time.Second
|
||||
|
||||
// TestStateStoreServerOptionsTolerateTheDialersKeepalive holds one watch open
|
||||
// against two servers: one with grpc's default enforcement policy, one with the
|
||||
// options the State Store process actually serves with. The default policy is
|
||||
// the defect -- a server that counts every one of the dialer's pings as a
|
||||
// strike tears down a connection that carries no traffic for ~40s, and a watch
|
||||
// is silent by design while the watched resource's version is unchanged
|
||||
// (internal/app/controlplane's Watch; the dashboard holds the same shape over
|
||||
// its own connection in internal/webui/api_control_plane.go, writing nothing
|
||||
// between versions). That case is also what keeps the other one honest: a
|
||||
// policy that never strikes has nothing to tear down.
|
||||
func TestStateStoreServerOptionsTolerateTheDialersKeepalive(t *testing.T) {
|
||||
// The options the standalone State Store serves its local profile with,
|
||||
// taken from the composition rather than hand-built here.
|
||||
served, loopback, err := stateStoreServerOpts("127.0.0.1:9090", "", &staterpc.TaskGrants{})
|
||||
if err != nil {
|
||||
t.Fatalf("stateStoreServerOpts: %v", err)
|
||||
}
|
||||
if !loopback {
|
||||
t.Fatal("127.0.0.1 must be reported as loopback")
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
opts []grpc.ServerOption
|
||||
tornDown bool
|
||||
}{
|
||||
{
|
||||
// The control: without an enforcement policy coherent with the
|
||||
// dialer, the same client on the same stream dies mid-window.
|
||||
name: "grpc's default enforcement policy tears the dialer's watch down",
|
||||
tornDown: true,
|
||||
},
|
||||
{
|
||||
name: "the state store's server options keep the dialer's watch up",
|
||||
opts: served,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
elapsed, recvErr := holdWatch(t, tc.opts, keepaliveWindow)
|
||||
// The server never writes, so the stream can only end because the
|
||||
// window ran out or because the connection was torn down under it.
|
||||
expired := streamEndedByWindow(recvErr)
|
||||
t.Logf("watch ended after %s with %v", elapsed, recvErr)
|
||||
if tc.tornDown {
|
||||
if recvErr == nil || expired {
|
||||
t.Fatalf("the dialer's watch was still open after %s: the default EnforcementPolicy (MinTime 5m) must strike pings that arrive 10s apart and answer with GOAWAY too_many_pings, so this case is not reaching the strike budget and the case below would prove nothing", keepaliveWindow)
|
||||
}
|
||||
if !strings.Contains(recvErr.Error(), "too_many_pings") {
|
||||
t.Fatalf("the dialer's watch ended after %s with %v: want GOAWAY too_many_pings, not some other connection loss", elapsed, recvErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
if recvErr == nil || !expired {
|
||||
t.Fatalf("the dialer's watch ended after %s: %v\nA watch carrying no traffic must survive on the pings staterpc.Dial sends; a server policy that strikes them answers a quiet connection with a dead one at ~40s, which is worse than the keepalive it was added for", elapsed, recvErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// quietWatch is the server half of a control-plane watch that never writes: it
|
||||
// holds one stream open and sends nothing, the state a watch is in while the
|
||||
// watched resource's version is unchanged.
|
||||
type quietWatch struct {
|
||||
controlpb.UnimplementedControlPlaneServiceServer
|
||||
started chan struct{}
|
||||
}
|
||||
|
||||
func (w *quietWatch) Watch(_ *controlpb.WatchRequest, stream controlpb.ControlPlaneService_WatchServer) error {
|
||||
close(w.started)
|
||||
<-stream.Context().Done()
|
||||
return stream.Context().Err()
|
||||
}
|
||||
|
||||
// holdWatch serves opts over bufconn, dials it with the shipped dialer
|
||||
// (staterpc.Dial, so the client keepalive is exactly the one the daemon, the
|
||||
// dashboard and the agent container run), holds one watch stream open for
|
||||
// window, and reports how long the stream lasted and how it ended.
|
||||
func holdWatch(t *testing.T, opts []grpc.ServerOption, window time.Duration) (time.Duration, error) {
|
||||
t.Helper()
|
||||
listener := bufconn.Listen(1 << 20)
|
||||
watch := &quietWatch{started: make(chan struct{})}
|
||||
server := grpc.NewServer(opts...)
|
||||
controlpb.RegisterControlPlaneServiceServer(server, watch)
|
||||
served := make(chan struct{})
|
||||
go func() {
|
||||
defer close(served)
|
||||
_ = server.Serve(listener)
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
server.Stop()
|
||||
_ = listener.Close()
|
||||
<-served
|
||||
})
|
||||
|
||||
client := dialStateStore(t, listener, restartAdminToken)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), window)
|
||||
defer cancel()
|
||||
stream, err := client.ControlPlane().Watch(ctx, &controlpb.WatchRequest{Kind: "any"})
|
||||
if err != nil {
|
||||
t.Fatalf("open watch: %v", err)
|
||||
}
|
||||
select {
|
||||
case <-watch.started:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the server never started the watch stream")
|
||||
}
|
||||
start := time.Now()
|
||||
// The server never writes, so this returns only when the window ends or the
|
||||
// connection dies.
|
||||
_, recvErr := stream.Recv()
|
||||
return time.Since(start), recvErr
|
||||
}
|
||||
|
||||
// streamEndedByWindow reports whether err is the window running out rather than
|
||||
// the connection being torn down under it.
|
||||
func streamEndedByWindow(err error) bool {
|
||||
return err != nil && (errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded)
|
||||
}
|
||||
@@ -25,8 +25,8 @@ func TestStateStoreServerOptsLoopbackIsInsecure(t *testing.T) {
|
||||
if !loopback {
|
||||
t.Fatal("loopback listener should report loopback")
|
||||
}
|
||||
if len(opts) != 0 {
|
||||
t.Fatalf("loopback listener should have no server opts, got %d", len(opts))
|
||||
if len(opts) != 1 {
|
||||
t.Fatalf("loopback listener should install only the keepalive enforcement policy (no token interceptors), got %d server opts", len(opts))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,8 +41,8 @@ func TestStateStoreServerOptsNonLoopbackRequiresToken(t *testing.T) {
|
||||
if loopback {
|
||||
t.Fatal("non-loopback listener should not report loopback")
|
||||
}
|
||||
if len(opts) != 2 {
|
||||
t.Fatalf("non-loopback listener should install two server options (the unary and stream task-grant interceptors), got %d", len(opts))
|
||||
if len(opts) != 3 {
|
||||
t.Fatalf("non-loopback listener should install three server options (the keepalive enforcement policy and the unary and stream task-grant interceptors), got %d", len(opts))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -89,6 +89,53 @@ func clientKeepaliveParams() keepalive.ClientParameters {
|
||||
}
|
||||
}
|
||||
|
||||
// serverKeepaliveMinTime is the server half of the same agreement: the
|
||||
// shortest gap between two pings a State Store server accepts before it counts
|
||||
// a strike. It is half the client's ping interval rather than a copy of it;
|
||||
// ServerKeepaliveOption carries the arithmetic.
|
||||
const serverKeepaliveMinTime = 5 * time.Second
|
||||
|
||||
// ServerKeepaliveOption returns the server half of the keepalive agreement
|
||||
// Dial's clients enter, for the composition that serves the State Store
|
||||
// contract to install on its grpc.Server (internal/app/archied's
|
||||
// stateStoreServerOpts installs it on every listener).
|
||||
//
|
||||
// The two sides have to agree on one number. A ping arriving less than
|
||||
// EnforcementPolicy.MinTime after the previous ping is a strike, and the server
|
||||
// answers the third strike with GOAWAY too_many_pings (maxPingStrikes is 2; a
|
||||
// write on that transport is what resets the counter). grpc's default policy
|
||||
// MinTime is 5 minutes while the client above pings every 10 seconds, so a
|
||||
// connection that carries no write for ~40s -- which is what a watch is while
|
||||
// the watched resource's version is unchanged -- is torn down where before the
|
||||
// client keepalive existed it stayed up. A process that writes on the same
|
||||
// transport every 30s (the apply-status re-stamp, applystatus.RestampInterval)
|
||||
// resets the counter often enough to hide this; an idle connection is what it
|
||||
// kills, and a keepalive that disconnects is worse than no keepalive.
|
||||
//
|
||||
// MinTime is 5s, half the client's ping interval. The client's interval is the
|
||||
// larger of its Time and grpc's KeepaliveMinPingTime clamp -- 10s, so no caller
|
||||
// can ask for faster pings -- and its keepalive loop arms its timer Time ahead
|
||||
// and re-arms it only later, on read activity, so consecutive pings are never
|
||||
// closer than ~10s apart. A 5s floor thus separates every ping from a strike by
|
||||
// a factor of two, worth more than Go's timer granularity and any scheduling
|
||||
// delay, which can only push pings further apart. A MinTime of 10s would sit
|
||||
// exactly on the boundary the clamp already enforces, where a ping a
|
||||
// microsecond early is a strike; 5 minutes is the defect itself.
|
||||
//
|
||||
// The floor is not a blanket exemption: a ping arriving less than 5s after the
|
||||
// previous one, and any ping on a connection with no stream, still strikes.
|
||||
// PermitWithoutStream stays false, matching the client, which never pings a
|
||||
// connection with no streams: every client this server has is a staterpc.Dial
|
||||
// -- the daemon's control-plane client, the archie-ui dashboard, archie-
|
||||
// messaging and the archie-agent container -- so permitting streamless pings
|
||||
// would only relax a setting none of them exercises.
|
||||
func ServerKeepaliveOption() grpc.ServerOption {
|
||||
return grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
|
||||
MinTime: serverKeepaliveMinTime,
|
||||
PermitWithoutStream: false,
|
||||
})
|
||||
}
|
||||
|
||||
// TargetIsLoopback reports whether addr's host is a loopback address. Both
|
||||
// literal loopback IPs (127.0.0.1, ::1) and the "localhost" hostname count;
|
||||
// everything else (including the wildcard 0.0.0.0 and DNS names) is treated
|
||||
|
||||
Reference in new issue
Block a user