diff --git a/internal/app/archied/control_plane_watch.go b/internal/app/archied/control_plane_watch.go index c50a53112..e7b99136d 100644 --- a/internal/app/archied/control_plane_watch.go +++ b/internal/app/archied/control_plane_watch.go @@ -12,23 +12,40 @@ import ( // retrying: a watch reconnects for as long as the process lives. // // A control plane that is down is retried slowly enough that it is not a -// request loop, and that holds for every shape the outage takes because the -// window an attempt is judged against is derived from the delay it would -// otherwise be charged (attemptHealthy): a store that errors at once and a -// store that stalls until its read times out are both charged, so both climb -// the ladder instead of sitting at the minimum. +// request loop, and the charge reaches the shapes of outage an attempt that +// ENDS can take: a store that errors at once is charged, and so is one that +// stalls for less than the window the attempt is judged against, because that +// window is derived from the delay the attempt would otherwise be charged and +// widens with the ladder instead of sitting at its floor (attemptHealthy). The +// reach stops at the window: a stall still going when the window closes is read +// as a store that answered, so an outage whose every attempt takes at least +// that long leaves the ladder at the minimum -- deliberately, and pinned by +// TestKeepWatchTreatsAStallPastTheWindowAsAStoreThatAnswered. +// +// This loop charges only attempts that end, and imposes no timeout of its own: +// nextUpdate waits on the stream and on the loop's own context, never on a +// clock, so a peer that accepts the stream and then never answers it ends +// nothing here and this code has nothing to charge. What ends that stream is +// the transport's client keepalive, +// internal/infrastructure/staterpc/dial.go (Time 10s, Timeout 5s): the +// unacknowledged PING tears the connection down, the watch stream errors, the +// attempt ends, and here it is charged like any other attempt that ended. A +// hung peer is that keepalive's business, not this loop's. The watch rides +// that connection because the daemon takes its control-plane client off the +// *staterpc.Client the dial returned, on the same connection the keepalive was +// installed on (internal/app/archied/bootstrap.go, openStateStoreAdapter, +// which reads client.ControlPlane()). const ( controlPlaneWatchRetryMin = 250 * time.Millisecond controlPlaneWatchRetryMax = 30 * time.Second ) -// controlPlaneWatchHealthyMultiple is how many times the delay an attempt would -// otherwise be charged an attempt that delivered nothing must outlive to be -// read as a store that answered and had nothing to say. It is strictly more -// than one, so that window is always wider than the charge itself: a window -// equal to the delay would read a store that fails as slowly as that delay as -// healthy on every attempt, which is the fixed-rate loop this rule exists to -// charge. +// controlPlaneWatchHealthyMultiple multiplies the delay an attempt would +// otherwise be charged into the window that attempt must stay open to be read +// as a store that answered and had nothing to say. It is strictly more than +// one, so that window is always wider than the charge itself: a window equal to +// the delay would read a store that fails as slowly as that delay as healthy on +// every attempt, which is the fixed-rate loop this rule exists to charge. const controlPlaneWatchHealthyMultiple = 2 // keepWatch delivers every update from first and, when that stream ends, @@ -67,14 +84,26 @@ const controlPlaneWatchHealthyMultiple = 2 // loop. // // The rule is one sentence: an attempt is healthy iff it delivered an update -// this watch had not handled, or it outlived a window that is a strict multiple -// of the delay it would otherwise be charged. The window is derived from that -// charge rather than fixed at the minimum, which is what makes the rule reach -// every shape of outage: a store that has stopped answering does not error at -// once, it stalls until its read times out, and an attempt that took longer to -// fail than a window fixed at the minimum would be read as a healthy store on -// every attempt, leaving the delay one minimum per timeout for as long as the -// store is gone. +// this watch had not handled, or it was still open when the window closed +// (attemptHealthy, controlPlaneWatchHealthyWindow -- the delay the attempt would +// otherwise be charged, multiplied by a factor strictly greater than one, so +// the window is always wider than that charge). The window is derived from that +// charge rather than fixed at the minimum, which is what carries the rule past +// the shape an immediate error does not cover: a store that has stopped +// answering does not error at once, it stalls, and an attempt that stalls for +// less than the window is charged and the delay climbs a rung. +// +// The window is not a read timeout, and this loop imposes none of its own (see +// controlPlaneWatchRetry for what ends an attempt against a peer that never +// answers), so the reach of the rule stops at the window itself: a stall still +// going when the window closes is read as a store that answered, and the delay +// stays at the minimum for as long as every attempt takes that long. That +// is deliberate rather than a gap to close with another rule. The stalled +// attempt lasts the window, so the reopens it leaves behind are bounded by the +// attempt and not by the delay -- about 0.8 a second at the minimum rung, +// against the four a second this rule exists to charge -- and a shape that slow +// is not the request loop it exists for. It is pinned by +// TestKeepWatchTreatsAStallPastTheWindowAsAStoreThatAnswered. // // open, versionOf and deliver are the kind's halves. open re-establishes the // stream -- the caller opens the first one itself, so that a control plane @@ -84,9 +113,12 @@ const controlPlaneWatchHealthyMultiple = 2 // document reaches the settings page. // // wait is the delay between a stream ending and the reopen that follows it, and -// production passes waitFor. It is injected so that the delay the loop actually -// waited is observable: the retry_in log line states the delay the loop charged -// and would agree with a loop that waited some other multiple of it. +// production passes waitFor. It is injected so that the delay the loop hands it +// is observable as a value: the retry_in log line states the delay the loop +// charged, and a loop that charged one delay and waited another would leave that +// attribute agreeing with itself. What the injected wait cannot show is a +// multiple taken inside waitFor itself, which is a timer for exactly the d it is +// given (TestKeepWatchWaitsTheDelayItCharged reads the argument, not the wait). func keepWatch[T any]( ctx context.Context, log *slog.Logger, @@ -138,21 +170,27 @@ func keepWatch[T any]( // attemptHealthy is what one attempt that ended makes of the reconnect delay. // An attempt is healthy iff it delivered an update this watch had not handled -// -- the store answered, whatever the stream did afterwards -- or it outlived -// controlPlaneWatchHealthyWindow, which is a store that had nothing to say and -// was still there. An attempt that did neither delivered nothing and died -// inside the window it would have been charged: the stream was accepted and the -// store was not answering, which is a failed attempt and the only thing the -// delay grows on. +// -- the store answered, whatever the stream did afterwards -- or it was still +// open when controlPlaneWatchHealthyWindow closed, at that boundary and not +// only past it, which is a store that had nothing to say and was still there. +// An attempt that did neither delivered nothing and died inside the window it +// would have been charged: the stream was accepted and the store was not +// answering, which is a failed attempt and the only thing the delay grows on. func attemptHealthy(progressed bool, openFor, backoff time.Duration) bool { return progressed || openFor >= controlPlaneWatchHealthyWindow(backoff) } // controlPlaneWatchHealthyWindow is how long an attempt that delivered nothing -// must stay open to count as a store that answered and had nothing to say: a -// strict multiple of the delay the attempt would otherwise be charged. The -// window moves with the ladder, so the charge reaches a store that fails slowly -// at every rung of it, which a window fixed at the minimum does not. +// must stay open to count as a store that answered and had nothing to say: the +// delay the attempt would otherwise be charged, multiplied by +// controlPlaneWatchHealthyMultiple. The window moves with the ladder, so the +// charge reaches a store that fails slowly at every rung of it, which a window +// fixed at the minimum does not. The multiple is strictly more than one, so the +// window is always wider than that charge -- a store that fails as slowly as +// the charge it would be paid is still charged, which is the shape of +// TestWatchRetryGrowsWhileTheStateStoreIsDown that says so -- and an attempt is +// healthy at the window as well as past it (TestAttemptHealthyAtTheWindowBoundary +// pins that edge). func controlPlaneWatchHealthyWindow(backoff time.Duration) time.Duration { return controlPlaneWatchHealthyMultiple * retryDelay(backoff, false) } diff --git a/internal/app/archied/control_plane_watch_test.go b/internal/app/archied/control_plane_watch_test.go index ac523d665..6b2527771 100644 --- a/internal/app/archied/control_plane_watch_test.go +++ b/internal/app/archied/control_plane_watch_test.go @@ -6,6 +6,7 @@ import ( "go/ast" "go/parser" "go/token" + "go/types" "io" "log/slog" "slices" @@ -574,11 +575,17 @@ func TestKeepWatchGuardrails(t *testing.T) { // TestKeepWatchChargesAnAttemptThatEndedWithNothing pins the reconnect delay // against the three ways one attempt can end, which is the whole of the // decision: an attempt that delivered an update of the store's is healthy even -// though the stream ended the instant after it, an attempt that outlived the -// window it would have been charged is healthy even though the store had -// nothing to say, and an attempt that did neither failed -- the stream was -// accepted and then died without the store having answered -- so the reconnect -// behind it is charged. +// though the stream ended the instant after it, an attempt that was still open +// when the window it would have been charged closed is healthy even though the +// store had nothing to say, and an attempt that did neither failed -- the +// stream was accepted and then died without the store having answered -- so the +// reconnect behind it is charged. +// +// The window case stands open past the window rather than for exactly it: the +// sleep in it starts before the loop's attempt does, so an attempt measured +// against the window it was started with is an attempt that can be judged a +// hair early. The edge itself -- the window closing exactly on the attempt's +// end -- is pinned without a clock by TestAttemptHealthyAtTheWindowBoundary. // // Every case reopens successfully, so a loop that reset the delay on the mere // acceptance of a stream would report the minimum for the failed attempt too: @@ -609,7 +616,7 @@ func TestKeepWatchChargesAnAttemptThatEndedWithNothing(t *testing.T) { stream := make(chan int64) go func() { defer close(stream) - time.Sleep(controlPlaneWatchHealthyWindow(controlPlaneWatchRetryMin)) + time.Sleep(controlPlaneWatchHealthyWindow(controlPlaneWatchRetryMin) + controlPlaneWatchRetryMin/10) }() return stream }, @@ -633,6 +640,88 @@ func TestKeepWatchChargesAnAttemptThatEndedWithNothing(t *testing.T) { } } +// TestKeepWatchTreatsAStallPastTheWindowAsAStoreThatAnswered pins the outer +// boundary of attemptHealthy where it is decided, and documents what the rule +// stops reaching: the derived window charges a stall shorter than itself and +// stops there. A stall still going when the window closes ends an attempt that +// was open for the whole window, so the rule reads it as a store that answered +// and had nothing to say, and the ladder is left at the minimum -- [250ms +// 250ms], not [250ms 500ms]. +// +// That is deliberate, and it is not a gap to close with a further rule. The +// stalled attempt itself lasts the window, so the reopens it leaves behind are +// under one a second at the minimum rung, against the four a second this rule +// exists to charge, and a shape that slow is not the request loop. What the +// boundary does mean is that an outage is charged by its failed attempts only +// while they fail faster than the window; one whose every attempt takes that +// long is reconnected at the minimum, and its RPC rate is bounded by the +// attempt rather than by the delay. +func TestKeepWatchTreatsAStallPastTheWindowAsAStoreThatAnswered(t *testing.T) { + t.Parallel() + log, retries := newRetryLog() + // The minimum rung judges an attempt against a window of four times the + // minimum (controlPlaneWatchHealthyWindow(controlPlaneWatchRetryMin)), and + // each attempt here stands open for five times the minimum -- a whole + // minimum past that window -- carrying nothing: a store that has stopped + // answering and never errors. The margin is a whole minimum rather than a + // hair, because the sleep starts before the loop's attempt does and a race + // with the window is not what this test is for. + stall := 5 * controlPlaneWatchRetryMin + stalled := func() <-chan int64 { + stream := make(chan int64) + go func() { + defer close(stream) + time.Sleep(stall) + }() + return stream + } + open := func(context.Context, int64) (<-chan int64, error) { return stalled(), nil } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go keepWatch(ctx, log, "test", 0, stalled(), open, waitFor, func(version int64) int64 { return version }, func(int64) {}) + + want := []time.Duration{controlPlaneWatchRetryMin, controlPlaneWatchRetryMin} + if got := awaitRetries(t, retries, len(want)); !slices.Equal(got[:len(want)], want) { + t.Errorf("reconnects waited %v, want %v: an attempt that stayed open for the whole %v window is read as a store that answered, so the ladder is left at the minimum and the reopen rate is bounded by the attempt itself rather than by the delay", + got[:len(want)], want, controlPlaneWatchHealthyWindow(controlPlaneWatchRetryMin)) + } + cancel() +} + +// TestAttemptHealthyAtTheWindowBoundary pins the edge where the sentence and +// the operator meet, which nothing else in the package does: the comparison is +// inclusive, so an attempt still open when the window closes -- at it, and not +// only past it -- is a store that answered, and an attempt a nanosecond inside +// it is charged. The window is a strictly wider multiple of the delay the +// attempt would otherwise be charged at every rung of the ladder, the cap +// included, so a store that fails as slowly as the charge it would be paid is +// still charged: an inclusive boundary does not read that store as healthy. +func TestAttemptHealthyAtTheWindowBoundary(t *testing.T) { + t.Parallel() + for _, backoff := range []time.Duration{controlPlaneWatchRetryMin, 3 * time.Second, controlPlaneWatchRetryMax} { + t.Run(backoff.String(), func(t *testing.T) { + t.Parallel() + charge := retryDelay(backoff, false) + window := controlPlaneWatchHealthyWindow(backoff) + if window <= charge { + t.Fatalf("the window at a %v charge is %v, want strictly wider than the charge: a window equal to it reads a store that fails as slowly as the charge as healthy on every attempt, which is the fixed-rate loop the window exists to charge", charge, window) + } + for _, tt := range []struct { + name string + openFor time.Duration + want bool + }{ + {"an attempt still open when the window closes is healthy", window, true}, + {"an attempt that died a nanosecond inside the window is charged", window - time.Nanosecond, false}, + } { + if got := attemptHealthy(false, tt.openFor, backoff); got != tt.want { + t.Errorf("attemptHealthy(progressed=false, openFor=%v, backoff=%v) = %t, want %t: %s", tt.openFor, backoff, got, tt.want, tt.name) + } + } + }) + } +} + // TestKeepWatchBackoffResetsAfterASuccessfulReopen pins the retry delay's other // half: it grows while attempts fail, and a reopen the store answers returns it // to the minimum. Without the reset a store that is up and quiet ratchets the @@ -1011,49 +1100,83 @@ func (s *busyStream) Recv() (*pb.WatchResponse, error) { return &pb.WatchResponse{Resource: s.resource}, nil } +// keepWatchWaitArgument is the position of keepWatch's wait parameter +// (ctx, log, kind, after, first, open, wait, versionOf, deliver): the injection +// point that makes the ladder observable, and so an argument whose value a call +// site could get wrong with no behavioural test behind it. +const keepWatchWaitArgument = 6 + // TestBothWatchStreamsRunThroughTheReconnectLoop pins the wiring the // behavioural tests drive one step below: both launch sites must hand their // stream to the one reconnect loop, or a kind keeps the single-shot goroutine -// that ends for the life of the process. Asserted on the source because +// that ends for the life of the process. What the loop is handed is pinned too, +// because a call site that reached keepWatch with a wait of its own would be a +// second schedule nothing else drives: the persona launch site's wait has no +// behavioural test of its own -- the outage test reads the delays one level +// down, and only for the settings launch site. Asserted on the source because // setupChatRuntime needs a fully built boot (stores, runtime, memory engines) // to run -- the technique composition_order_test.go and // TestSetupGatewayChatCarriesStatusHealth already use for wiring nothing at // runtime observes. func TestBothWatchStreamsRunThroughTheReconnectLoop(t *testing.T) { fileset := token.NewFileSet() - for _, tt := range []struct{ file, holder, call string }{ - {file: "control_plane.go", holder: "startWorkflowExecutionSettings", call: "keepWatch"}, + for _, tt := range []struct { + file, holder, call string + // wait is the name the call must hand keepWatch in its wait position, + // empty for a call that is not keepWatch itself. + wait string + }{ + {file: "control_plane.go", holder: "startWorkflowExecutionSettings", call: "keepWatch", wait: "waitFor"}, {file: "gateway_runtime.go", holder: "setupChatRuntime", call: "watchPersonas"}, - {file: "gateway_runtime.go", holder: "watchPersonas", call: "keepWatch"}, + {file: "gateway_runtime.go", holder: "watchPersonas", call: "keepWatch", wait: "waitFor"}, } { parsed, err := parser.ParseFile(fileset, tt.file, nil, 0) if err != nil { t.Fatalf("parse %s: %v", tt.file, err) } - if !callsName(methodBody(t, parsed, tt.holder), tt.call) { + call := callTo(methodBody(t, parsed, tt.holder), tt.call) + if call == nil { t.Errorf("%s never calls %s: that watch stream would not be re-established", tt.holder, tt.call) + continue + } + if tt.wait == "" { + continue + } + if len(call.Args) <= keepWatchWaitArgument { + t.Errorf("%s calls %s with %d arguments, want at least %d: the wait cannot be read, so nothing here pins which wait the loop is handed", tt.holder, tt.call, len(call.Args), keepWatchWaitArgument+1) + continue + } + if got := types.ExprString(call.Args[keepWatchWaitArgument]); got != tt.wait { + t.Errorf("%s hands %s %s as its wait, want %s: the loop's backoff has to be the wait that reports a cancelled context and is the injection point the ladder is read through, not a schedule of the call site's own", tt.holder, tt.call, got, tt.wait) } } } -// callsName reports whether body calls name, as a plain function or as a -// method of anything. -func callsName(body *ast.BlockStmt, name string) bool { - called := false +// callTo returns body's first call to name, as a plain function or as a method +// of anything, or nil when body does not call it. +func callTo(body *ast.BlockStmt, name string) *ast.CallExpr { + var found *ast.CallExpr ast.Inspect(body, func(node ast.Node) bool { + if found != nil { + return false + } call, ok := node.(*ast.CallExpr) if !ok { return true } switch fun := call.Fun.(type) { case *ast.Ident: - called = called || fun.Name == name + if fun.Name == name { + found = call + } case *ast.SelectorExpr: - called = called || fun.Sel.Name == name + if fun.Sel.Name == name { + found = call + } } return true }) - return called + return found } // retryLog is a slog.Handler that records the retry delay of each reconnect,