From c798f89a35581edf0803734183daf10cc0c97d75 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 01:51:11 +1000 Subject: [PATCH 1/2] fix(archieui): exit cleanly when graceful shutdown overruns its deadline The gate failed intermittently with "task: Failed to run task \"test\": exit status 1" and a tailed log that hid the failing package. The package was cmd/archie-ui, which sorts before internal/logging and so never appeared in the visible tail. Cause: serve() returned the error from server.Shutdown, which is context.DeadlineExceeded whenever a request is still draining when ShutdownTimeout elapses. That converted a normal load-induced overrun into a non-zero process exit, and TestUIProcessServesTheDashboardAgainstLiveDependencies asserts a clean exit after SIGTERM. Fix at the producer, in serve(): on overrun, force-close the remaining connections and report a clean shutdown. No timeout is lengthened, no retry, skip or -p 1 is added, and the smoke test's assertion is untouched. Reproduced under parallel build load: -count=30 on the suspect test produced the exact shutdown-deadline signature once; after the fix, 30/30 pass with zero occurrences. The new test drives the deadline path deterministically by holding a request in flight rather than relying on machine load. --- internal/app/archieui/run.go | 13 +++++- internal/app/archieui/serve_test.go | 64 +++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 internal/app/archieui/serve_test.go diff --git a/internal/app/archieui/run.go b/internal/app/archieui/run.go index 39f546432..ad97e8b99 100644 --- a/internal/app/archieui/run.go +++ b/internal/app/archieui/run.go @@ -81,7 +81,9 @@ func Run(ctx context.Context, options Options) error { // serve runs handler on listener until ctx ends, then shuts the HTTP server // down within ShutdownTimeout so in-flight dashboard requests finish before -// the contract clients close. +// the contract clients close. A graceful shutdown that overruns its deadline +// is a normal shutdown under load, not a process failure: the remaining +// connections are force-closed and serve reports clean. func serve(ctx context.Context, listener net.Listener, handler http.Handler, opts Options) error { server := &http.Server{Handler: handler, ReadHeaderTimeout: opts.ReadHeaderTimeout} serveErr := make(chan error, 1) @@ -90,7 +92,14 @@ func serve(ctx context.Context, listener net.Listener, handler http.Handler, opt case <-ctx.Done(): shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), opts.ShutdownTimeout) defer cancel() - return server.Shutdown(shutdownCtx) + if err := server.Shutdown(shutdownCtx); err != nil { + // Shutdown returned once the drain deadline elapsed with a request + // still in flight. Force-close what is left so shutdown finishes + // and the process exits cleanly rather than carrying a deadline + // error to the operator as a fatal exit (archie-core-u4xu). + _ = server.Close() + } + return nil case err := <-serveErr: if errors.Is(err, http.ErrServerClosed) { return nil diff --git a/internal/app/archieui/serve_test.go b/internal/app/archieui/serve_test.go new file mode 100644 index 000000000..917e775db --- /dev/null +++ b/internal/app/archieui/serve_test.go @@ -0,0 +1,64 @@ +package archieui + +import ( + "context" + "net" + "net/http" + "testing" + "time" +) + +// TestServeForceClosesWhenGracefulShutdownOverruns pins the shutdown behaviour +// behind the process smoke test (TestUIProcessServesTheDashboardAgainstLiveDependencies): +// when a graceful shutdown overruns its deadline because an in-flight request +// is still draining, serve must force-close the remaining connections and +// report a clean (nil) shutdown rather than a context.DeadlineExceeded that +// makes the process exit non-zero (archie-core-u4xu). The request is held open +// deterministically rather than by machine load, so the deadline path is +// driven directly. +func TestServeForceClosesWhenGracefulShutdownOverruns(t *testing.T) { + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + started := make(chan struct{}) + release := make(chan struct{}) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + select { + case <-release: + case <-r.Context().Done(): + } + }) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + t.Cleanup(func() { close(release) }) + + done := make(chan error, 1) + go func() { done <- serve(ctx, listener, handler, Options{ShutdownTimeout: 50 * time.Millisecond}) }() + + go func() { + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+listener.Addr().String(), nil) + if err != nil { + return + } + resp, err := http.DefaultClient.Do(req) + if err == nil { + _ = resp.Body.Close() + } + }() + <-started + + cancel() + + select { + case err := <-done: + if err != nil { + t.Fatalf("serve returned %v after graceful shutdown overran its deadline, want a clean shutdown", err) + } + case <-time.After(5 * time.Second): + t.Fatal("serve did not return after the shutdown deadline overran") + } +} From 01d2548476000fb97a3824cbbcaab4add81dfb71 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 02:12:27 +1000 Subject: [PATCH 2/2] test(archieui): make the shutdown regression test table-driven and bounded Review fixes for PR #876. The production fix is unchanged; this reworks only the test. - Bound the wait for the handler-start signal and select on the request goroutine's error and on an early serve() exit, so a request that never reaches the handler fails promptly instead of hanging until the package-wide timeout. - Restructure into a table, per AGENTS.md:256 / CLAUDE.md:256 ("table-driven tests against target behaviour"): a request that drains within ShutdownTimeout returns nil, and a request still in flight when it elapses is force-closed and still returns nil. The overrun row was verified to fail against the previous "return server.Shutdown(shutdownCtx)" and to pass against the approved force-close. Both rows hold the request in flight deterministically, so no case depends on machine load. --- internal/app/archieui/serve_test.go | 140 +++++++++++++++++++--------- 1 file changed, 95 insertions(+), 45 deletions(-) diff --git a/internal/app/archieui/serve_test.go b/internal/app/archieui/serve_test.go index 917e775db..6e057d408 100644 --- a/internal/app/archieui/serve_test.go +++ b/internal/app/archieui/serve_test.go @@ -4,61 +4,111 @@ import ( "context" "net" "net/http" + "sync" "testing" "time" ) -// TestServeForceClosesWhenGracefulShutdownOverruns pins the shutdown behaviour -// behind the process smoke test (TestUIProcessServesTheDashboardAgainstLiveDependencies): -// when a graceful shutdown overruns its deadline because an in-flight request -// is still draining, serve must force-close the remaining connections and -// report a clean (nil) shutdown rather than a context.DeadlineExceeded that -// makes the process exit non-zero (archie-core-u4xu). The request is held open -// deterministically rather than by machine load, so the deadline path is -// driven directly. -func TestServeForceClosesWhenGracefulShutdownOverruns(t *testing.T) { - listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("listen: %v", err) +// TestServeReportsCleanShutdown pins the shutdown behaviour behind the process +// smoke test (TestUIProcessServesTheDashboardAgainstLiveDependencies): serve +// must report a clean (nil) shutdown both when a graceful shutdown drains its +// in-flight request within ShutdownTimeout and when the deadline overruns and +// the remaining connection is force-closed (archie-core-u4xu). The request is +// held open deterministically rather than by machine load, so each deadline +// path is driven directly instead of depending on scheduling. +func TestServeReportsCleanShutdown(t *testing.T) { + tests := []struct { + name string + shutdownTimeout time.Duration + // holdPastTimeout keeps the request in flight until serve force-closes + // it, instead of releasing it as soon as shutdown begins. + holdPastTimeout bool + }{ + { + name: "request drains within ShutdownTimeout", + shutdownTimeout: 5 * time.Second, + }, + { + name: "request still in flight when ShutdownTimeout elapses", + shutdownTimeout: 50 * time.Millisecond, + holdPastTimeout: true, + }, } - started := make(chan struct{}) - release := make(chan struct{}) - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - close(started) - select { - case <-release: - case <-r.Context().Done(): - } - }) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - t.Cleanup(func() { close(release) }) + started := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseRequest := func() { releaseOnce.Do(func() { close(release) }) } - done := make(chan error, 1) - go func() { done <- serve(ctx, listener, handler, Options{ShutdownTimeout: 50 * time.Millisecond}) }() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + select { + case <-release: + case <-r.Context().Done(): + } + }) - go func() { - req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+listener.Addr().String(), nil) - if err != nil { - return - } - resp, err := http.DefaultClient.Do(req) - if err == nil { - _ = resp.Body.Close() - } - }() - <-started + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + t.Cleanup(releaseRequest) - cancel() + done := make(chan error, 1) + go func() { done <- serve(ctx, listener, handler, Options{ShutdownTimeout: tt.shutdownTimeout}) }() - select { - case err := <-done: - if err != nil { - t.Fatalf("serve returned %v after graceful shutdown overran its deadline, want a clean shutdown", err) - } - case <-time.After(5 * time.Second): - t.Fatal("serve did not return after the shutdown deadline overran") + reqErr := make(chan error, 1) + go func() { + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+listener.Addr().String(), nil) + if err != nil { + reqErr <- err + return + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + reqErr <- err + return + } + _ = resp.Body.Close() + reqErr <- nil + }() + + // Wait for the request to reach the handler. A request that never + // lands must fail promptly rather than hang until the package + // timeout, and a serve that exits before the request lands must + // also fail rather than deadlock. + select { + case <-started: + case err := <-reqErr: + t.Fatalf("request did not reach the handler: %v", err) + case err := <-done: + if err == nil { + t.Fatal("serve returned cleanly before the request landed") + } + t.Fatalf("serve returned before the request landed: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("request did not reach the handler before the wait deadline") + } + + cancel() + + if !tt.holdPastTimeout { + releaseRequest() + } + + select { + case err := <-done: + if err != nil { + t.Fatalf("serve returned %v after graceful shutdown, want a clean shutdown", err) + } + case <-time.After(5 * time.Second): + t.Fatal("serve did not return after shutdown") + } + }) } }