From a237a355323b7bf22c08071e625703debb9dd8d8 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 15 Aug 2026 12:53:31 +1000 Subject: [PATCH] fix(agentworker): back off before refetching a nak'd stage message On a stage-handle failure, the main loop logged the error, called Nak (which requests redelivery), and looped straight back into FetchStage with no delay -- unlike the fetch-error branch three lines above, which already backs off for a second. A deterministic failure (a broken gate, a malformed request) therefore refetched and failed again immediately, flooding the log with one "handle failed" line per loop iteration instead of per genuine retry -- hundreds of identical lines within the same second in production. Adds the same time.Second backoff after Nak. --- internal/app/agentworker/stage_loop.go | 5 +++++ internal/app/agentworker/stage_loop_test.go | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/app/agentworker/stage_loop.go b/internal/app/agentworker/stage_loop.go index d52d17d9d..489408927 100644 --- a/internal/app/agentworker/stage_loop.go +++ b/internal/app/agentworker/stage_loop.go @@ -43,6 +43,11 @@ func runMainLoopWithDelay(ctx context.Context, bus stageBus, log *slog.Logger, d if err := message.Nak(); err != nil { log.Warn("nak failed", "err", err) } + // A Nak requests redelivery. Without a backoff, a deterministic + // failure (a broken gate, a malformed request) refetches and + // fails again immediately, flooding the log with one line per + // loop iteration instead of one per genuine retry. + delay(time.Second) continue } if err := message.Ack(); err != nil { diff --git a/internal/app/agentworker/stage_loop_test.go b/internal/app/agentworker/stage_loop_test.go index fcfc37a10..cdba81124 100644 --- a/internal/app/agentworker/stage_loop_test.go +++ b/internal/app/agentworker/stage_loop_test.go @@ -148,7 +148,7 @@ func TestRunMainLoopNaksHandleErrorsAndContinues(t *testing.T) { message := &stageMessageStub{requestErr: errors.New("decode request: invalid"), nakErr: test.nakErr} bus := &stageBusStub{fetches: []stageFetch{{message: message}, {err: errors.New("cancelled"), before: cancel}}} log, output := stageLoopLogger() - delay, _ := noDelay(t) + delay, delays := noDelay(t) runMainLoopWithDelay(ctx, bus, log, delay) if message.nakCalls != 1 || message.ackCalls != 0 { t.Fatalf("nak/ack = (%d,%d), want (1,0)", message.nakCalls, message.ackCalls) @@ -156,6 +156,13 @@ func TestRunMainLoopNaksHandleErrorsAndContinues(t *testing.T) { if got := strings.Contains(output.String(), "nak failed"); got != test.wantNakLog { t.Fatalf("nak failure logged = %v, want %v", got, test.wantNakLog) } + // A Nak requests redelivery, so without a backoff the very next + // FetchStage hands back the same failing message immediately -- + // a deterministic failure (e.g. a broken gate) then spins as a + // tight log-flooding loop instead of backing off. + if len(*delays) != 1 || (*delays)[0] != time.Second { + t.Fatalf("delays = %v, want a single %v backoff before refetching a nak'd message", *delays, time.Second) + } }) } }