fix(archieui): exit cleanly when graceful shutdown overruns its deadline
The gate failed intermittently with "task: Failed to run task \"test\": exit status 1" and a tailed log that hid the failing package. The package was cmd/archie-ui, which sorts before internal/logging and so never appeared in the visible tail. Cause: serve() returned the error from server.Shutdown, which is context.DeadlineExceeded whenever a request is still draining when ShutdownTimeout elapses. That converted a normal load-induced overrun into a non-zero process exit, and TestUIProcessServesTheDashboardAgainstLiveDependencies asserts a clean exit after SIGTERM. Fix at the producer, in serve(): on overrun, force-close the remaining connections and report a clean shutdown. No timeout is lengthened, no retry, skip or -p 1 is added, and the smoke test's assertion is untouched. Reproduced under parallel build load: -count=30 on the suspect test produced the exact shutdown-deadline signature once; after the fix, 30/30 pass with zero occurrences. The new test drives the deadline path deterministically by holding a request in flight rather than relying on machine load.
This commit is contained in:
1 parent
121fbf730f
commit
c798f89a35
2 files changed
+75
-2
No files matched your search
@@ -81,7 +81,9 @@ func Run(ctx context.Context, options Options) error {
|
||||
|
||||
// serve runs handler on listener until ctx ends, then shuts the HTTP server
|
||||
// down within ShutdownTimeout so in-flight dashboard requests finish before
|
||||
// the contract clients close.
|
||||
// the contract clients close. A graceful shutdown that overruns its deadline
|
||||
// is a normal shutdown under load, not a process failure: the remaining
|
||||
// connections are force-closed and serve reports clean.
|
||||
func serve(ctx context.Context, listener net.Listener, handler http.Handler, opts Options) error {
|
||||
server := &http.Server{Handler: handler, ReadHeaderTimeout: opts.ReadHeaderTimeout}
|
||||
serveErr := make(chan error, 1)
|
||||
@@ -90,7 +92,14 @@ func serve(ctx context.Context, listener net.Listener, handler http.Handler, opt
|
||||
case <-ctx.Done():
|
||||
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), opts.ShutdownTimeout)
|
||||
defer cancel()
|
||||
return server.Shutdown(shutdownCtx)
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
// Shutdown returned once the drain deadline elapsed with a request
|
||||
// still in flight. Force-close what is left so shutdown finishes
|
||||
// and the process exits cleanly rather than carrying a deadline
|
||||
// error to the operator as a fatal exit (archie-core-u4xu).
|
||||
_ = server.Close()
|
||||
}
|
||||
return nil
|
||||
case err := <-serveErr:
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package archieui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestServeForceClosesWhenGracefulShutdownOverruns pins the shutdown behaviour
|
||||
// behind the process smoke test (TestUIProcessServesTheDashboardAgainstLiveDependencies):
|
||||
// when a graceful shutdown overruns its deadline because an in-flight request
|
||||
// is still draining, serve must force-close the remaining connections and
|
||||
// report a clean (nil) shutdown rather than a context.DeadlineExceeded that
|
||||
// makes the process exit non-zero (archie-core-u4xu). The request is held open
|
||||
// deterministically rather than by machine load, so the deadline path is
|
||||
// driven directly.
|
||||
func TestServeForceClosesWhenGracefulShutdownOverruns(t *testing.T) {
|
||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
close(started)
|
||||
select {
|
||||
case <-release:
|
||||
case <-r.Context().Done():
|
||||
}
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
t.Cleanup(func() { close(release) })
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- serve(ctx, listener, handler, Options{ShutdownTimeout: 50 * time.Millisecond}) }()
|
||||
|
||||
go func() {
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://"+listener.Addr().String(), nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err == nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
}()
|
||||
<-started
|
||||
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("serve returned %v after graceful shutdown overran its deadline, want a clean shutdown", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("serve did not return after the shutdown deadline overran")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user