Merge pull request #844 from samcharles93/fix/archied-watchdog-test-flake

fix(archied): stop the shutdown-watchdog test racing the scheduler
This commit is contained in:
sam authored and GitHub committed 2026-09-17 21:24:29 +10:00
commit 8ab86b86b3
1 file changed
+37 -3
+37 -3
View File
@@ -11,6 +11,24 @@ import (
// period -- far longer than anything a unit test should wait for.
const testWatchdogLeash = 30 * time.Millisecond
// testWatchdogExitTimeout bounds how long a test waits for the watchdog's
// forced exit to be observed. It is deliberately generous: the leash only
// starts the timer, and the firing goroutine must then be scheduled onto a
// core before the injected exit is observable. A deadline of a few leash
// lengths is a wall-clock race against the Go scheduler, and it fails
// intermittently on a loaded machine -- the test then parks real work behind a
// gate failure that has nothing to do with the change under test. The test
// still completes in ~30ms; this is a ceiling, not a sleep.
const testWatchdogExitTimeout = 5 * time.Second
// testWatchdogStandDownPeriod is the shorter wait used where the test asserts
// the ABSENCE of a forced exit. Load cannot make that assertion lie: a slow
// runner only delays the very exit the test is checking for, so a tight bound
// is safe in this direction and keeps the suite fast. It still leaves the
// leash many times over for a broken disarm to reveal itself, which is the
// regression this half of the pair exists to catch.
const testWatchdogStandDownPeriod = testWatchdogLeash * 10
func discardWatchdogLogger() *slog.Logger {
return slog.New(slog.DiscardHandler)
}
@@ -19,6 +37,13 @@ func discardWatchdogLogger() *slog.Logger {
// graceful shutdown begins and never completes (the shutdown machinery is
// hung, so the disarm is never called), the watchdog force-exits the process
// after the leash expires.
//
// The timing contract asserted here is the one that actually matters and the
// one scheduling jitter cannot falsify: the exit must NOT happen before the
// leash, and it must happen. Waiting for it with a generous ceiling keeps the
// second half honest under load; asserting elapsed >= leash keeps the first
// half strict, because delaying the goroutine only ever makes it later, never
// earlier.
func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) {
exited := make(chan int, 1)
w := newShutdownWatchdog(testWatchdogLeash, discardWatchdogLogger())
@@ -28,6 +53,7 @@ func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) {
_ = w.watch(began)
// Graceful shutdown begins and never completes: no disarm is called.
start := time.Now()
close(began)
select {
@@ -35,14 +61,22 @@ func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) {
if code != 1 {
t.Fatalf("watchdog force-exited with code %d, want 1", code)
}
case <-time.After(testWatchdogLeash * 2):
t.Fatalf("watchdog did not force-exit within leash %s", testWatchdogLeash)
if elapsed := time.Since(start); elapsed < testWatchdogLeash {
t.Fatalf("watchdog force-exited after %s, before the %s leash had expired", elapsed, testWatchdogLeash)
}
case <-time.After(testWatchdogExitTimeout):
t.Fatalf("watchdog did not force-exit within %s (leash %s)", testWatchdogExitTimeout, testWatchdogLeash)
}
}
// TestShutdownWatchdogCompletingWithinLeashDoesNotExit proves the inverse:
// a shutdown that completes within the leash (the disarm is called before the
// leash expires) does not trigger the watchdog.
//
// The disarm closes the watchdog's done channel, so the goroutine stands down
// and no exit can follow. This asserts the ABSENCE of a forced exit, which is
// the direction load cannot falsify -- a slow runner only delays the exit
// being checked for -- so a tight bound is both safe and honest here.
func TestShutdownWatchdogCompletingWithinLeashDoesNotExit(t *testing.T) {
exited := make(chan int, 1)
w := newShutdownWatchdog(testWatchdogLeash, discardWatchdogLogger())
@@ -58,7 +92,7 @@ func TestShutdownWatchdogCompletingWithinLeashDoesNotExit(t *testing.T) {
select {
case code := <-exited:
t.Fatalf("watchdog force-exited with code %d despite completing within leash", code)
case <-time.After(testWatchdogLeash * 3):
case <-time.After(testWatchdogStandDownPeriod):
// Correct: a shutdown that completed within the leash did not
// trigger the forced exit.
}