Merge pull request #876 from samcharles93/fix/flaky-ui-shutdown-deadline

fix(archieui): exit cleanly when graceful shutdown overruns its deadline
This commit is contained in:
sam authored and GitHub committed 2026-09-19 02:13:59 +10:00
commit 7df5279fb3
2 files changed
+125 -2

No files matched your search

+11 -2
View File
@@ -81,7 +81,9 @@ func Run(ctx context.Context, options Options) error {
// serve runs handler on listener until ctx ends, then shuts the HTTP server
// down within ShutdownTimeout so in-flight dashboard requests finish before
// the contract clients close.
// the contract clients close. A graceful shutdown that overruns its deadline
// is a normal shutdown under load, not a process failure: the remaining
// connections are force-closed and serve reports clean.
func serve(ctx context.Context, listener net.Listener, handler http.Handler, opts Options) error {
server := &http.Server{Handler: handler, ReadHeaderTimeout: opts.ReadHeaderTimeout}
serveErr := make(chan error, 1)
@@ -90,7 +92,14 @@ func serve(ctx context.Context, listener net.Listener, handler http.Handler, opt
case <-ctx.Done():
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), opts.ShutdownTimeout)
defer cancel()
return server.Shutdown(shutdownCtx)
if err := server.Shutdown(shutdownCtx); err != nil {
// Shutdown returned once the drain deadline elapsed with a request
// still in flight. Force-close what is left so shutdown finishes
// and the process exits cleanly rather than carrying a deadline
// error to the operator as a fatal exit (archie-core-u4xu).
_ = server.Close()
}
return nil
case err := <-serveErr:
if errors.Is(err, http.ErrServerClosed) {
return nil
+114
View File
@@ -0,0 +1,114 @@
package archieui
import (
"context"
"net"
"net/http"
"sync"
"testing"
"time"
)
// TestServeReportsCleanShutdown pins the shutdown behaviour behind the process
// smoke test (TestUIProcessServesTheDashboardAgainstLiveDependencies): serve
// must report a clean (nil) shutdown both when a graceful shutdown drains its
// in-flight request within ShutdownTimeout and when the deadline overruns and
// the remaining connection is force-closed (archie-core-u4xu). The request is
// held open deterministically rather than by machine load, so each deadline
// path is driven directly instead of depending on scheduling.
func TestServeReportsCleanShutdown(t *testing.T) {
tests := []struct {
name string
shutdownTimeout time.Duration
// holdPastTimeout keeps the request in flight until serve force-closes
// it, instead of releasing it as soon as shutdown begins.
holdPastTimeout bool
}{
{
name: "request drains within ShutdownTimeout",
shutdownTimeout: 5 * time.Second,
},
{
name: "request still in flight when ShutdownTimeout elapses",
shutdownTimeout: 50 * time.Millisecond,
holdPastTimeout: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
started := make(chan struct{})
release := make(chan struct{})
var releaseOnce sync.Once
releaseRequest := func() { releaseOnce.Do(func() { close(release) }) }
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(started)
select {
case <-release:
case <-r.Context().Done():
}
})
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
t.Cleanup(releaseRequest)
done := make(chan error, 1)
go func() { done <- serve(ctx, listener, handler, Options{ShutdownTimeout: tt.shutdownTimeout}) }()
reqErr := make(chan error, 1)
go func() {
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+listener.Addr().String(), nil)
if err != nil {
reqErr <- err
return
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
reqErr <- err
return
}
_ = resp.Body.Close()
reqErr <- nil
}()
// Wait for the request to reach the handler. A request that never
// lands must fail promptly rather than hang until the package
// timeout, and a serve that exits before the request lands must
// also fail rather than deadlock.
select {
case <-started:
case err := <-reqErr:
t.Fatalf("request did not reach the handler: %v", err)
case err := <-done:
if err == nil {
t.Fatal("serve returned cleanly before the request landed")
}
t.Fatalf("serve returned before the request landed: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("request did not reach the handler before the wait deadline")
}
cancel()
if !tt.holdPastTimeout {
releaseRequest()
}
select {
case err := <-done:
if err != nil {
t.Fatalf("serve returned %v after graceful shutdown, want a clean shutdown", err)
}
case <-time.After(5 * time.Second):
t.Fatal("serve did not return after shutdown")
}
})
}
}