Merge pull request #844 from samcharles93/fix/archied-watchdog-test-flake
fix(archied): stop the shutdown-watchdog test racing the scheduler
This commit is contained in:
commit
8ab86b86b3
1 file changed
+37
-3
@@ -11,6 +11,24 @@ import (
|
||||
// period -- far longer than anything a unit test should wait for.
|
||||
const testWatchdogLeash = 30 * time.Millisecond
|
||||
|
||||
// testWatchdogExitTimeout bounds how long a test waits for the watchdog's
|
||||
// forced exit to be observed. It is deliberately generous: the leash only
|
||||
// starts the timer, and the firing goroutine must then be scheduled onto a
|
||||
// core before the injected exit is observable. A deadline of a few leash
|
||||
// lengths is a wall-clock race against the Go scheduler, and it fails
|
||||
// intermittently on a loaded machine -- the test then parks real work behind a
|
||||
// gate failure that has nothing to do with the change under test. The test
|
||||
// still completes in ~30ms; this is a ceiling, not a sleep.
|
||||
const testWatchdogExitTimeout = 5 * time.Second
|
||||
|
||||
// testWatchdogStandDownPeriod is the shorter wait used where the test asserts
|
||||
// the ABSENCE of a forced exit. Load cannot make that assertion lie: a slow
|
||||
// runner only delays the very exit the test is checking for, so a tight bound
|
||||
// is safe in this direction and keeps the suite fast. It still leaves the
|
||||
// leash many times over for a broken disarm to reveal itself, which is the
|
||||
// regression this half of the pair exists to catch.
|
||||
const testWatchdogStandDownPeriod = testWatchdogLeash * 10
|
||||
|
||||
func discardWatchdogLogger() *slog.Logger {
|
||||
return slog.New(slog.DiscardHandler)
|
||||
}
|
||||
@@ -19,6 +37,13 @@ func discardWatchdogLogger() *slog.Logger {
|
||||
// graceful shutdown begins and never completes (the shutdown machinery is
|
||||
// hung, so the disarm is never called), the watchdog force-exits the process
|
||||
// after the leash expires.
|
||||
//
|
||||
// The timing contract asserted here is the one that actually matters and the
|
||||
// one scheduling jitter cannot falsify: the exit must NOT happen before the
|
||||
// leash, and it must happen. Waiting for it with a generous ceiling keeps the
|
||||
// second half honest under load; asserting elapsed >= leash keeps the first
|
||||
// half strict, because delaying the goroutine only ever makes it later, never
|
||||
// earlier.
|
||||
func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) {
|
||||
exited := make(chan int, 1)
|
||||
w := newShutdownWatchdog(testWatchdogLeash, discardWatchdogLogger())
|
||||
@@ -28,6 +53,7 @@ func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) {
|
||||
_ = w.watch(began)
|
||||
|
||||
// Graceful shutdown begins and never completes: no disarm is called.
|
||||
start := time.Now()
|
||||
close(began)
|
||||
|
||||
select {
|
||||
@@ -35,14 +61,22 @@ func TestShutdownWatchdogHangingPastLeashForcesExit(t *testing.T) {
|
||||
if code != 1 {
|
||||
t.Fatalf("watchdog force-exited with code %d, want 1", code)
|
||||
}
|
||||
case <-time.After(testWatchdogLeash * 2):
|
||||
t.Fatalf("watchdog did not force-exit within leash %s", testWatchdogLeash)
|
||||
if elapsed := time.Since(start); elapsed < testWatchdogLeash {
|
||||
t.Fatalf("watchdog force-exited after %s, before the %s leash had expired", elapsed, testWatchdogLeash)
|
||||
}
|
||||
case <-time.After(testWatchdogExitTimeout):
|
||||
t.Fatalf("watchdog did not force-exit within %s (leash %s)", testWatchdogExitTimeout, testWatchdogLeash)
|
||||
}
|
||||
}
|
||||
|
||||
// TestShutdownWatchdogCompletingWithinLeashDoesNotExit proves the inverse:
|
||||
// a shutdown that completes within the leash (the disarm is called before the
|
||||
// leash expires) does not trigger the watchdog.
|
||||
//
|
||||
// The disarm closes the watchdog's done channel, so the goroutine stands down
|
||||
// and no exit can follow. This asserts the ABSENCE of a forced exit, which is
|
||||
// the direction load cannot falsify -- a slow runner only delays the exit
|
||||
// being checked for -- so a tight bound is both safe and honest here.
|
||||
func TestShutdownWatchdogCompletingWithinLeashDoesNotExit(t *testing.T) {
|
||||
exited := make(chan int, 1)
|
||||
w := newShutdownWatchdog(testWatchdogLeash, discardWatchdogLogger())
|
||||
@@ -58,7 +92,7 @@ func TestShutdownWatchdogCompletingWithinLeashDoesNotExit(t *testing.T) {
|
||||
select {
|
||||
case code := <-exited:
|
||||
t.Fatalf("watchdog force-exited with code %d despite completing within leash", code)
|
||||
case <-time.After(testWatchdogLeash * 3):
|
||||
case <-time.After(testWatchdogStandDownPeriod):
|
||||
// Correct: a shutdown that completed within the leash did not
|
||||
// trigger the forced exit.
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user