fix(agentworker): back off before refetching a nak'd stage message
On a stage-handle failure, the main loop logged the error, called Nak (which requests redelivery), and looped straight back into FetchStage with no delay -- unlike the fetch-error branch three lines above, which already backs off for a second. A deterministic failure (a broken gate, a malformed request) therefore refetched and failed again immediately, flooding the log with one "handle failed" line per loop iteration instead of per genuine retry -- hundreds of identical lines within the same second in production. Adds the same time.Second backoff after Nak.
This commit is contained in:
1 parent
95ca1cad6d
commit
a237a35532
2 files changed
+13
-1
No files matched your search
@@ -43,6 +43,11 @@ func runMainLoopWithDelay(ctx context.Context, bus stageBus, log *slog.Logger, d
|
||||
if err := message.Nak(); err != nil {
|
||||
log.Warn("nak failed", "err", err)
|
||||
}
|
||||
// A Nak requests redelivery. Without a backoff, a deterministic
|
||||
// failure (a broken gate, a malformed request) refetches and
|
||||
// fails again immediately, flooding the log with one line per
|
||||
// loop iteration instead of one per genuine retry.
|
||||
delay(time.Second)
|
||||
continue
|
||||
}
|
||||
if err := message.Ack(); err != nil {
|
||||
|
||||
@@ -148,7 +148,7 @@ func TestRunMainLoopNaksHandleErrorsAndContinues(t *testing.T) {
|
||||
message := &stageMessageStub{requestErr: errors.New("decode request: invalid"), nakErr: test.nakErr}
|
||||
bus := &stageBusStub{fetches: []stageFetch{{message: message}, {err: errors.New("cancelled"), before: cancel}}}
|
||||
log, output := stageLoopLogger()
|
||||
delay, _ := noDelay(t)
|
||||
delay, delays := noDelay(t)
|
||||
runMainLoopWithDelay(ctx, bus, log, delay)
|
||||
if message.nakCalls != 1 || message.ackCalls != 0 {
|
||||
t.Fatalf("nak/ack = (%d,%d), want (1,0)", message.nakCalls, message.ackCalls)
|
||||
@@ -156,6 +156,13 @@ func TestRunMainLoopNaksHandleErrorsAndContinues(t *testing.T) {
|
||||
if got := strings.Contains(output.String(), "nak failed"); got != test.wantNakLog {
|
||||
t.Fatalf("nak failure logged = %v, want %v", got, test.wantNakLog)
|
||||
}
|
||||
// A Nak requests redelivery, so without a backoff the very next
|
||||
// FetchStage hands back the same failing message immediately --
|
||||
// a deterministic failure (e.g. a broken gate) then spins as a
|
||||
// tight log-flooding loop instead of backing off.
|
||||
if len(*delays) != 1 || (*delays)[0] != time.Second {
|
||||
t.Fatalf("delays = %v, want a single %v backoff before refetching a nak'd message", *delays, time.Second)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user