From a8018ac06ce7a97ef3d5831b337e60a89f6fd8f1 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 17 Sep 2026 11:17:19 +0000 Subject: [PATCH] fix(archied): stop the shutdown-watchdog test racing the scheduler TestShutdownWatchdogHangingPastLeashForcesExit gave the watchdog two leash lengths (60ms) to force an exit, but the 30ms leash only starts the timer -- the firing goroutine must then be scheduled before the injected exit is observable. Measured margin is ~29.7ms, so the test is a wall-clock race against the Go scheduler and fails intermittently on a loaded machine, where it parks real work behind a gate failure unrelated to the change under test. The assertion is now the contract that actually matters and that scheduling jitter cannot falsify: the exit must not happen before the leash, and it must happen. Waiting for it uses a generous ceiling, and the new elapsed >= leash check catches an early fire strictly -- something the old test could not do, because it only bounded lateness. The inverse test asserts the ABSENCE of a forced exit, which load cannot falsify (a slow runner only delays the exit being checked for), so it keeps a tight bound and the suite stays fast. Verified by mutation: removing the forced exit fails the first test, and firing before the leash now fails it on the new elapsed check. --- .../app/archied/shutdown_watchdog_test.go | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/internal/app/archied/shutdown_watchdog_test.go b/internal/app/archied/shutdown_watchdog_test.go index c9b119cd8..c72b2f852 100644 --- a/internal/app/archied/shutdown_watchdog_test.go +++ b/internal/app/archied/shutdown_watchdog_test.go @@ -11,6 +11,24 @@ import ( // period -- far longer than anything a unit test should wait for. const testWatchdogLeash = 30 * time.Millisecond +// testWatchdogExitTimeout bounds how long a test waits for the watchdog's +// forced exit to be observed. It is deliberately generous: the leash only +// starts the timer, and the firing goroutine must then be scheduled onto a +// core before the injected exit is observable. A deadline of a few leash +// lengths is a wall-clock race against the Go scheduler, and it fails +// intermittently on a loaded machine -- the test then parks real work behind a +// gate failure that has nothing to do with the change under test. The test +// still completes in ~30ms; this is a ceiling, not a sleep. +const testWatchdogExitTimeout = 5 * time.Second + +// testWatchdogStandDownPeriod is the shorter wait used where the test asserts +// the ABSENCE of a forced exit. Load cannot make that assertion lie: a slow +// runner only delays the very exit the test is checking for, so a tight bound +// is safe in this direction and keeps the suite fast. It still leaves the +// leash many times over for a broken disarm to reveal itself, which is the +// regression this half of the pair exists to catch. +const testWatchdogStandDownPeriod = testWatchdogLeash * 10 + func discardWatchdogLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } @@ -19,6 +37,13 @@ func discardWatchdogLogger() *slog.Logger { // graceful shutdown begins and never completes (the shutdown machinery is // hung, so the disarm is never called), the watchdog force-exits the process // after the leash expires. +// +// The timing contract asserted here is the one that actually matters and the +// one scheduling jitter cannot falsify: the exit must NOT happen before the +// leash, and it must happen. Waiting for it with a generous ceiling keeps the +// second half honest under load; asserting elapsed >= leash keeps the first +// half strict, because delaying the goroutine only ever makes it later, never +// earlier. func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) { exited := make(chan int, 1) w := newShutdownWatchdog(testWatchdogLeash, discardWatchdogLogger()) @@ -28,6 +53,7 @@ func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) { _ = w.watch(began) // Graceful shutdown begins and never completes: no disarm is called. + start := time.Now() close(began) select { @@ -35,14 +61,22 @@ func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) { if code != 1 { t.Fatalf("watchdog force-exited with code %d, want 1", code) } - case <-time.After(testWatchdogLeash * 2): - t.Fatalf("watchdog did not force-exit within leash %s", testWatchdogLeash) + if elapsed := time.Since(start); elapsed < testWatchdogLeash { + t.Fatalf("watchdog force-exited after %s, before the %s leash had expired", elapsed, testWatchdogLeash) + } + case <-time.After(testWatchdogExitTimeout): + t.Fatalf("watchdog did not force-exit within %s (leash %s)", testWatchdogExitTimeout, testWatchdogLeash) } } // TestShutdownWatchdogCompletingWithinLeashDoesNotExit proves the inverse: // a shutdown that completes within the leash (the disarm is called before the // leash expires) does not trigger the watchdog. +// +// The disarm closes the watchdog's done channel, so the goroutine stands down +// and no exit can follow. This asserts the ABSENCE of a forced exit, which is +// the direction load cannot falsify -- a slow runner only delays the exit +// being checked for -- so a tight bound is both safe and honest here. func TestShutdownWatchdogCompletingWithinLeashDoesNotExit(t *testing.T) { exited := make(chan int, 1) w := newShutdownWatchdog(testWatchdogLeash, discardWatchdogLogger()) @@ -58,7 +92,7 @@ func TestShutdownWatchdogCompletingWithinLeashDoesNotExit(t *testing.T) { select { case code := <-exited: t.Fatalf("watchdog force-exited with code %d despite completing within leash", code) - case <-time.After(testWatchdogLeash * 3): + case <-time.After(testWatchdogStandDownPeriod): // Correct: a shutdown that completed within the leash did not // trigger the forced exit. }