fix(container): stop a late max-uptime callback resurrecting teardown state

This commit is contained in:
sam committed 2026-09-22 01:57:59 +10:00
1 parent 477b3738a3
commit da6e49a3be
2 files changed
+153 -30

No files matched your search

+77 -30
View File
@@ -257,38 +257,81 @@ type containerTeardown struct {
// lifetime cap elapses. The timer never touches p.active: Release (or Close)
// remains the only owner of the active slot.
//
// The reaper never calls forgetTeardown itself: Release is always coming for
// this container (the pool guarantees every acquired container is eventually
// released) and is the sole owner of deleting the map entry. Two callers
// deleting it independently is what let a freshly-recreated, unclosed entry
// slip between the reaper's delete and Release's claim check.
// The entry and its timer are recorded in one critical section, so the
// callback cannot run ahead of the bookkeeping it needs: time.AfterFunc hands
// control to the callback as soon as the duration elapses, and the callback's
// first act is claimReaper, which takes p.mu. Holding p.mu across both the
// entry creation and the timer's creation is what makes that claim observe a
// container that is armed rather than one that is mid-arm -- a timer recorded
// after a callback had already fired (and a Release already forgotten the
// entry) left an unclosed entry behind, with a one-shot timer that would never
// fire again.
//
// The lock covers only bookkeeping: the callback runs on its own goroutine and
// nothing here waits on it, and every Docker call it makes happens after it
// has released p.mu.
func (p *Pool) armMaxUptime(ctx context.Context, id string) {
timer := time.AfterFunc(p.cfg.MaxUptime, func() {
if !p.claimTeardown(id) {
return
}
zero := 0
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil {
p.log.Warn("max uptime stop failed", "id", id[:12], "err", err)
}
if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil {
p.log.Warn("max uptime remove failed", "id", id[:12], "err", err)
}
})
p.mu.Lock()
defer p.mu.Unlock()
p.teardownLocked(id).timer = timer
t := p.teardownLocked(id)
t.timer = time.AfterFunc(p.cfg.MaxUptime, func() {
p.reapMaxUptime(ctx, id)
})
}
// claimTeardown reports whether the caller owns this container's Docker
// teardown. The first caller wins; every later one is told to stay out.
func (p *Pool) claimTeardown(id string) bool {
// reapMaxUptime is the max-uptime callback body: it claims the container's
// Docker teardown and, if it won, stops and removes the container.
func (p *Pool) reapMaxUptime(ctx context.Context, id string) {
if !p.claimReaper(id) {
return
}
zero := 0
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil {
p.log.Warn("max uptime stop failed", "id", id[:12], "err", err)
}
if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil {
p.log.Warn("max uptime remove failed", "id", id[:12], "err", err)
}
}
// claimReaper reports whether the max-uptime callback owns this container's
// Docker teardown. The first claim wins; every later one is told to stay out.
//
// A missing entry is a loss, never a fresh claim: only armMaxUptime creates
// entries, so no entry means MaxUptime was never armed or Release has already
// forgotten the container -- in both cases the reaper has nothing to tear
// down. Creating one here is exactly how a callback that was still running
// when Release forgot the entry could resurrect it.
func (p *Pool) claimReaper(id string) bool {
p.mu.Lock()
defer p.mu.Unlock()
t := p.teardownLocked(id)
t := p.teardowns[id]
if t == nil || t.closed {
return false
}
t.closed = true
return true
}
// claimRelease reports whether Release owns this container's Docker teardown.
//
// Unlike the reaper, Release is the path that always exists: MaxUptime == 0
// arms no reaper at all, so the absence of an entry must still be a win here.
// The pool guarantees every acquired container is released exactly once, so by
// the time Release runs an entry can be missing only because a reaper was
// never armed -- an absent entry is never a live reaper in flight, since
// nothing but Release's forgetTeardown deletes one, and that runs after this
// claim.
func (p *Pool) claimRelease(id string) bool {
p.mu.Lock()
defer p.mu.Unlock()
t := p.teardowns[id]
if t == nil {
return true
}
if t.closed {
return false
}
@@ -297,9 +340,11 @@ func (p *Pool) claimTeardown(id string) bool {
}
// forgetTeardown disarms the reaper and drops the container's bookkeeping, so
// the map does not grow for the life of the pool. Only Release calls this
// (see armMaxUptime); it is the container's final teardown step, so nothing
// else can race a fresh entry back into existence afterward.
// the map does not grow for the life of the pool. Only Release calls this, as
// the last step of a teardown it claimed, and deleting the entry is final: the
// reaper's claimReaper looks the entry up and never creates one, so a callback
// that was already running when the entry went away finds nothing to claim
// instead of racing a fresh entry back into existence.
func (p *Pool) forgetTeardown(id string) {
p.mu.Lock()
t := p.teardowns[id]
@@ -311,7 +356,9 @@ func (p *Pool) forgetTeardown(id string) {
}
// teardownLocked returns the container's teardown state, creating it on first
// use. Callers must hold p.mu.
// use. armMaxUptime is its only caller: entries are created by the producer of
// the reaper, under the same lock that lets the callback claim them, and never
// by a path that might be racing Release. Callers must hold p.mu.
func (p *Pool) teardownLocked(id string) *containerTeardown {
if p.teardowns == nil {
p.teardowns = make(map[string]*containerTeardown)
@@ -363,7 +410,7 @@ func (p *Pool) Release(ctx context.Context, c *Container) {
// period above; claiming before the sleep would quietly extend the cap by
// GracePeriod. Losing the claim means the reaper already removed this
// container, so there is nothing left to stop.
if p.claimTeardown(c.ID) {
if p.claimRelease(c.ID) {
// Detach from ctx before stopping. Release runs on the way out of a
// task, and the most important reason a task is on its way out is
// that it was cancelled -- at which point ctx is already dead, the
+76
View File
@@ -560,6 +560,82 @@ func TestReleaseAfterTheReaperFiredDoesNotTearDownTwice(t *testing.T) {
}
}
// A max-uptime callback that has already begun cannot be unwound by
// Timer.Stop, so it can reach its claim after Release has disarmed the timer
// and deleted the teardown entry. Release has torn the container down by then:
// the late callback must find nothing to claim, rather than recreating an
// unclosed entry behind Release's back and running stop/remove a second time
// against a container that is already gone.
//
// The interleaving is driven by running the reaper's callback body after
// Release returned. That is the closest reachable form: nothing in the public
// API lets a caller park the callback between the timer firing and its claim --
// the window is the callback goroutine's scheduling -- so a test that merely
// started a timer and a Release would be a coin flip. This covers the
// claim/forget ordering and the Docker calls that follow it; it does not cover
// the timer machinery itself.
func TestReaperCallbackAfterReleaseDoesNotResurrectTeardown(t *testing.T) {
const containerID = "9a8b7c6d5e4f32109a8b7c6d5e4f32109a8b7c6d5e4f32109a8b7c6d5e4f3210"
var stopCalls, removeCalls atomic.Int32
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasSuffix(r.URL.Path, "/containers/create"):
writeDockerJSON(t, w, map[string]any{"Id": containerID, "Warnings": []string{}})
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/start"):
w.WriteHeader(http.StatusNoContent)
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/stop"):
stopCalls.Add(1)
w.WriteHeader(http.StatusNoContent)
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID):
removeCalls.Add(1)
w.WriteHeader(http.StatusNoContent)
default:
http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound)
}
}))
t.Cleanup(dockerAPI.Close)
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = dockerClient.Close() })
pool := &Pool{
cli: dockerClient,
cfg: Config{Image: "test/image", MaxUptime: time.Hour},
log: discardLogger(),
}
c, err := pool.Acquire(context.Background(), nil, nil)
if err != nil {
t.Fatalf("Acquire: %v", err)
}
pool.Release(context.Background(), c)
stopAfterRelease, removeAfterRelease := stopCalls.Load(), removeCalls.Load()
if stopAfterRelease != 1 || removeAfterRelease != 1 {
t.Fatalf("Release stopped the container %d times and removed it %d times, want 1/1: a late callback proves nothing unless the release already did the teardown", stopAfterRelease, removeAfterRelease)
}
pool.reapMaxUptime(context.Background(), c.ID)
if got := stopCalls.Load(); got != stopAfterRelease {
t.Errorf("stop called %d times after Release, want %d: the late reaper tore down an already-released container", got, stopAfterRelease)
}
if got := removeCalls.Load(); got != removeAfterRelease {
t.Errorf("remove called %d times after Release, want %d: the late reaper tore down an already-released container", got, removeAfterRelease)
}
pool.mu.Lock()
remaining := len(pool.teardowns)
pool.mu.Unlock()
if remaining != 0 {
t.Errorf("pool holds %d teardown entr(ies) after Release, want 0: the late reaper recreated the entry", remaining)
}
}
// TestPoolActiveReportsInFlightContainers pins the worker-pool read surface
// /status reports as its container line: how many containers this pool
// currently holds, and the concurrency cap it enforces.