fix(agentworker): back off before refetching a nak'd stage message

On a stage-handle failure, the main loop logged the error, called
Nak (which requests redelivery), and looped straight back into
FetchStage with no delay -- unlike the fetch-error branch three lines
above, which already backs off for a second. A deterministic failure
(a broken gate, a malformed request) therefore refetched and failed
again immediately, flooding the log with one "handle failed" line per
loop iteration instead of per genuine retry -- hundreds of identical
lines within the same second in production.

Adds the same time.Second backoff after Nak.
This commit is contained in:
sam committed 2026-08-15 12:53:31 +10:00
1 parent 95ca1cad6d
commit a237a35532
2 files changed
+13 -1

No files matched your search

+5
View File
@@ -43,6 +43,11 @@ func runMainLoopWithDelay(ctx context.Context, bus stageBus, log *slog.Logger, d
if err := message.Nak(); err != nil {
log.Warn("nak failed", "err", err)
}
// A Nak requests redelivery. Without a backoff, a deterministic
// failure (a broken gate, a malformed request) refetches and
// fails again immediately, flooding the log with one line per
// loop iteration instead of one per genuine retry.
delay(time.Second)
continue
}
if err := message.Ack(); err != nil {
+8 -1
View File
@@ -148,7 +148,7 @@ func TestRunMainLoopNaksHandleErrorsAndContinues(t *testing.T) {
message := &stageMessageStub{requestErr: errors.New("decode request: invalid"), nakErr: test.nakErr}
bus := &stageBusStub{fetches: []stageFetch{{message: message}, {err: errors.New("cancelled"), before: cancel}}}
log, output := stageLoopLogger()
delay, _ := noDelay(t)
delay, delays := noDelay(t)
runMainLoopWithDelay(ctx, bus, log, delay)
if message.nakCalls != 1 || message.ackCalls != 0 {
t.Fatalf("nak/ack = (%d,%d), want (1,0)", message.nakCalls, message.ackCalls)
@@ -156,6 +156,13 @@ func TestRunMainLoopNaksHandleErrorsAndContinues(t *testing.T) {
if got := strings.Contains(output.String(), "nak failed"); got != test.wantNakLog {
t.Fatalf("nak failure logged = %v, want %v", got, test.wantNakLog)
}
// A Nak requests redelivery, so without a backoff the very next
// FetchStage hands back the same failing message immediately --
// a deterministic failure (e.g. a broken gate) then spins as a
// tight log-flooding loop instead of backing off.
if len(*delays) != 1 || (*delays)[0] != time.Second {
t.Fatalf("delays = %v, want a single %v backoff before refetching a nak'd message", *delays, time.Second)
}
})
}
}