diff --git a/internal/container/pool.go b/internal/container/pool.go index 134672191..a390859b3 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -257,38 +257,81 @@ type containerTeardown struct { // lifetime cap elapses. The timer never touches p.active: Release (or Close) // remains the only owner of the active slot. // -// The reaper never calls forgetTeardown itself: Release is always coming for -// this container (the pool guarantees every acquired container is eventually -// released) and is the sole owner of deleting the map entry. Two callers -// deleting it independently is what let a freshly-recreated, unclosed entry -// slip between the reaper's delete and Release's claim check. +// The entry and its timer are recorded in one critical section, so the +// callback cannot run ahead of the bookkeeping it needs: time.AfterFunc hands +// control to the callback as soon as the duration elapses, and the callback's +// first act is claimReaper, which takes p.mu. Holding p.mu across both the +// entry creation and the timer's creation is what makes that claim observe a +// container that is armed rather than one that is mid-arm -- a timer recorded +// after a callback had already fired (and a Release already forgotten the +// entry) left an unclosed entry behind, with a one-shot timer that would never +// fire again. +// +// The lock covers only bookkeeping: the callback runs on its own goroutine and +// nothing here waits on it, and every Docker call it makes happens after it +// has released p.mu. func (p *Pool) armMaxUptime(ctx context.Context, id string) { - timer := time.AfterFunc(p.cfg.MaxUptime, func() { - if !p.claimTeardown(id) { - return - } - zero := 0 - stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() - if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil { - p.log.Warn("max uptime stop failed", "id", id[:12], "err", err) - } - if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil { - p.log.Warn("max uptime remove failed", "id", id[:12], "err", err) - } - }) - p.mu.Lock() defer p.mu.Unlock() - p.teardownLocked(id).timer = timer + + t := p.teardownLocked(id) + t.timer = time.AfterFunc(p.cfg.MaxUptime, func() { + p.reapMaxUptime(ctx, id) + }) } -// claimTeardown reports whether the caller owns this container's Docker -// teardown. The first caller wins; every later one is told to stay out. -func (p *Pool) claimTeardown(id string) bool { +// reapMaxUptime is the max-uptime callback body: it claims the container's +// Docker teardown and, if it won, stops and removes the container. +func (p *Pool) reapMaxUptime(ctx context.Context, id string) { + if !p.claimReaper(id) { + return + } + zero := 0 + stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil { + p.log.Warn("max uptime stop failed", "id", id[:12], "err", err) + } + if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil { + p.log.Warn("max uptime remove failed", "id", id[:12], "err", err) + } +} + +// claimReaper reports whether the max-uptime callback owns this container's +// Docker teardown. The first claim wins; every later one is told to stay out. +// +// A missing entry is a loss, never a fresh claim: only armMaxUptime creates +// entries, so no entry means MaxUptime was never armed or Release has already +// forgotten the container -- in both cases the reaper has nothing to tear +// down. Creating one here is exactly how a callback that was still running +// when Release forgot the entry could resurrect it. +func (p *Pool) claimReaper(id string) bool { p.mu.Lock() defer p.mu.Unlock() - t := p.teardownLocked(id) + t := p.teardowns[id] + if t == nil || t.closed { + return false + } + t.closed = true + return true +} + +// claimRelease reports whether Release owns this container's Docker teardown. +// +// Unlike the reaper, Release is the path that always exists: MaxUptime == 0 +// arms no reaper at all, so the absence of an entry must still be a win here. +// The pool guarantees every acquired container is released exactly once, so by +// the time Release runs an entry can be missing only because a reaper was +// never armed -- an absent entry is never a live reaper in flight, since +// nothing but Release's forgetTeardown deletes one, and that runs after this +// claim. +func (p *Pool) claimRelease(id string) bool { + p.mu.Lock() + defer p.mu.Unlock() + t := p.teardowns[id] + if t == nil { + return true + } if t.closed { return false } @@ -297,9 +340,11 @@ func (p *Pool) claimTeardown(id string) bool { } // forgetTeardown disarms the reaper and drops the container's bookkeeping, so -// the map does not grow for the life of the pool. Only Release calls this -// (see armMaxUptime); it is the container's final teardown step, so nothing -// else can race a fresh entry back into existence afterward. +// the map does not grow for the life of the pool. Only Release calls this, as +// the last step of a teardown it claimed, and deleting the entry is final: the +// reaper's claimReaper looks the entry up and never creates one, so a callback +// that was already running when the entry went away finds nothing to claim +// instead of racing a fresh entry back into existence. func (p *Pool) forgetTeardown(id string) { p.mu.Lock() t := p.teardowns[id] @@ -311,7 +356,9 @@ func (p *Pool) forgetTeardown(id string) { } // teardownLocked returns the container's teardown state, creating it on first -// use. Callers must hold p.mu. +// use. armMaxUptime is its only caller: entries are created by the producer of +// the reaper, under the same lock that lets the callback claim them, and never +// by a path that might be racing Release. Callers must hold p.mu. func (p *Pool) teardownLocked(id string) *containerTeardown { if p.teardowns == nil { p.teardowns = make(map[string]*containerTeardown) @@ -363,7 +410,7 @@ func (p *Pool) Release(ctx context.Context, c *Container) { // period above; claiming before the sleep would quietly extend the cap by // GracePeriod. Losing the claim means the reaper already removed this // container, so there is nothing left to stop. - if p.claimTeardown(c.ID) { + if p.claimRelease(c.ID) { // Detach from ctx before stopping. Release runs on the way out of a // task, and the most important reason a task is on its way out is // that it was cancelled -- at which point ctx is already dead, the diff --git a/internal/container/pool_test.go b/internal/container/pool_test.go index 7269bc286..dd0f70676 100644 --- a/internal/container/pool_test.go +++ b/internal/container/pool_test.go @@ -560,6 +560,82 @@ func TestReleaseAfterTheReaperFiredDoesNotTearDownTwice(t *testing.T) { } } +// A max-uptime callback that has already begun cannot be unwound by +// Timer.Stop, so it can reach its claim after Release has disarmed the timer +// and deleted the teardown entry. Release has torn the container down by then: +// the late callback must find nothing to claim, rather than recreating an +// unclosed entry behind Release's back and running stop/remove a second time +// against a container that is already gone. +// +// The interleaving is driven by running the reaper's callback body after +// Release returned. That is the closest reachable form: nothing in the public +// API lets a caller park the callback between the timer firing and its claim -- +// the window is the callback goroutine's scheduling -- so a test that merely +// started a timer and a Release would be a coin flip. This covers the +// claim/forget ordering and the Docker calls that follow it; it does not cover +// the timer machinery itself. +func TestReaperCallbackAfterReleaseDoesNotResurrectTeardown(t *testing.T) { + const containerID = "9a8b7c6d5e4f32109a8b7c6d5e4f32109a8b7c6d5e4f32109a8b7c6d5e4f3210" + + var stopCalls, removeCalls atomic.Int32 + dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/containers/create"): + writeDockerJSON(t, w, map[string]any{"Id": containerID, "Warnings": []string{}}) + case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/start"): + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/stop"): + stopCalls.Add(1) + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/containers/"+containerID): + removeCalls.Add(1) + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(dockerAPI.Close) + + dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = dockerClient.Close() }) + + pool := &Pool{ + cli: dockerClient, + cfg: Config{Image: "test/image", MaxUptime: time.Hour}, + log: discardLogger(), + } + + c, err := pool.Acquire(context.Background(), nil, nil) + if err != nil { + t.Fatalf("Acquire: %v", err) + } + pool.Release(context.Background(), c) + + stopAfterRelease, removeAfterRelease := stopCalls.Load(), removeCalls.Load() + if stopAfterRelease != 1 || removeAfterRelease != 1 { + t.Fatalf("Release stopped the container %d times and removed it %d times, want 1/1: a late callback proves nothing unless the release already did the teardown", stopAfterRelease, removeAfterRelease) + } + + pool.reapMaxUptime(context.Background(), c.ID) + + if got := stopCalls.Load(); got != stopAfterRelease { + t.Errorf("stop called %d times after Release, want %d: the late reaper tore down an already-released container", got, stopAfterRelease) + } + if got := removeCalls.Load(); got != removeAfterRelease { + t.Errorf("remove called %d times after Release, want %d: the late reaper tore down an already-released container", got, removeAfterRelease) + } + + pool.mu.Lock() + remaining := len(pool.teardowns) + pool.mu.Unlock() + if remaining != 0 { + t.Errorf("pool holds %d teardown entr(ies) after Release, want 0: the late reaper recreated the entry", remaining) + } +} + // TestPoolActiveReportsInFlightContainers pins the worker-pool read surface // /status reports as its container line: how many containers this pool // currently holds, and the concurrency cap it enforces.