fix(container): stop a late max-uptime callback resurrecting teardown state
This commit is contained in:
1 parent
c579f6f021
commit
c9a4f4b7e1
2 files changed
+153
-30
No files matched your search
+77
-30
@@ -257,38 +257,81 @@ type containerTeardown struct {
|
||||
// lifetime cap elapses. The timer never touches p.active: Release (or Close)
|
||||
// remains the only owner of the active slot.
|
||||
//
|
||||
// The reaper never calls forgetTeardown itself: Release is always coming for
|
||||
// this container (the pool guarantees every acquired container is eventually
|
||||
// released) and is the sole owner of deleting the map entry. Two callers
|
||||
// deleting it independently is what let a freshly-recreated, unclosed entry
|
||||
// slip between the reaper's delete and Release's claim check.
|
||||
// The entry and its timer are recorded in one critical section, so the
|
||||
// callback cannot run ahead of the bookkeeping it needs: time.AfterFunc hands
|
||||
// control to the callback as soon as the duration elapses, and the callback's
|
||||
// first act is claimReaper, which takes p.mu. Holding p.mu across both the
|
||||
// entry creation and the timer's creation is what makes that claim observe a
|
||||
// container that is armed rather than one that is mid-arm -- a timer recorded
|
||||
// after a callback had already fired (and a Release already forgotten the
|
||||
// entry) left an unclosed entry behind, with a one-shot timer that would never
|
||||
// fire again.
|
||||
//
|
||||
// The lock covers only bookkeeping: the callback runs on its own goroutine and
|
||||
// nothing here waits on it, and every Docker call it makes happens after it
|
||||
// has released p.mu.
|
||||
func (p *Pool) armMaxUptime(ctx context.Context, id string) {
|
||||
timer := time.AfterFunc(p.cfg.MaxUptime, func() {
|
||||
if !p.claimTeardown(id) {
|
||||
return
|
||||
}
|
||||
zero := 0
|
||||
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil {
|
||||
p.log.Warn("max uptime stop failed", "id", id[:12], "err", err)
|
||||
}
|
||||
if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil {
|
||||
p.log.Warn("max uptime remove failed", "id", id[:12], "err", err)
|
||||
}
|
||||
})
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.teardownLocked(id).timer = timer
|
||||
|
||||
t := p.teardownLocked(id)
|
||||
t.timer = time.AfterFunc(p.cfg.MaxUptime, func() {
|
||||
p.reapMaxUptime(ctx, id)
|
||||
})
|
||||
}
|
||||
|
||||
// claimTeardown reports whether the caller owns this container's Docker
|
||||
// teardown. The first caller wins; every later one is told to stay out.
|
||||
func (p *Pool) claimTeardown(id string) bool {
|
||||
// reapMaxUptime is the max-uptime callback body: it claims the container's
|
||||
// Docker teardown and, if it won, stops and removes the container.
|
||||
func (p *Pool) reapMaxUptime(ctx context.Context, id string) {
|
||||
if !p.claimReaper(id) {
|
||||
return
|
||||
}
|
||||
zero := 0
|
||||
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := p.cli.ContainerStop(stopCtx, id, client.ContainerStopOptions{Timeout: &zero}); err != nil {
|
||||
p.log.Warn("max uptime stop failed", "id", id[:12], "err", err)
|
||||
}
|
||||
if _, err := p.cli.ContainerRemove(context.WithoutCancel(ctx), id, client.ContainerRemoveOptions{Force: true}); err != nil {
|
||||
p.log.Warn("max uptime remove failed", "id", id[:12], "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// claimReaper reports whether the max-uptime callback owns this container's
|
||||
// Docker teardown. The first claim wins; every later one is told to stay out.
|
||||
//
|
||||
// A missing entry is a loss, never a fresh claim: only armMaxUptime creates
|
||||
// entries, so no entry means MaxUptime was never armed or Release has already
|
||||
// forgotten the container -- in both cases the reaper has nothing to tear
|
||||
// down. Creating one here is exactly how a callback that was still running
|
||||
// when Release forgot the entry could resurrect it.
|
||||
func (p *Pool) claimReaper(id string) bool {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
t := p.teardownLocked(id)
|
||||
t := p.teardowns[id]
|
||||
if t == nil || t.closed {
|
||||
return false
|
||||
}
|
||||
t.closed = true
|
||||
return true
|
||||
}
|
||||
|
||||
// claimRelease reports whether Release owns this container's Docker teardown.
|
||||
//
|
||||
// Unlike the reaper, Release is the path that always exists: MaxUptime == 0
|
||||
// arms no reaper at all, so the absence of an entry must still be a win here.
|
||||
// The pool guarantees every acquired container is released exactly once, so by
|
||||
// the time Release runs an entry can be missing only because a reaper was
|
||||
// never armed -- an absent entry is never a live reaper in flight, since
|
||||
// nothing but Release's forgetTeardown deletes one, and that runs after this
|
||||
// claim.
|
||||
func (p *Pool) claimRelease(id string) bool {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
t := p.teardowns[id]
|
||||
if t == nil {
|
||||
return true
|
||||
}
|
||||
if t.closed {
|
||||
return false
|
||||
}
|
||||
@@ -297,9 +340,11 @@ func (p *Pool) claimTeardown(id string) bool {
|
||||
}
|
||||
|
||||
// forgetTeardown disarms the reaper and drops the container's bookkeeping, so
|
||||
// the map does not grow for the life of the pool. Only Release calls this
|
||||
// (see armMaxUptime); it is the container's final teardown step, so nothing
|
||||
// else can race a fresh entry back into existence afterward.
|
||||
// the map does not grow for the life of the pool. Only Release calls this, as
|
||||
// the last step of a teardown it claimed, and deleting the entry is final: the
|
||||
// reaper's claimReaper looks the entry up and never creates one, so a callback
|
||||
// that was already running when the entry went away finds nothing to claim
|
||||
// instead of racing a fresh entry back into existence.
|
||||
func (p *Pool) forgetTeardown(id string) {
|
||||
p.mu.Lock()
|
||||
t := p.teardowns[id]
|
||||
@@ -311,7 +356,9 @@ func (p *Pool) forgetTeardown(id string) {
|
||||
}
|
||||
|
||||
// teardownLocked returns the container's teardown state, creating it on first
|
||||
// use. Callers must hold p.mu.
|
||||
// use. armMaxUptime is its only caller: entries are created by the producer of
|
||||
// the reaper, under the same lock that lets the callback claim them, and never
|
||||
// by a path that might be racing Release. Callers must hold p.mu.
|
||||
func (p *Pool) teardownLocked(id string) *containerTeardown {
|
||||
if p.teardowns == nil {
|
||||
p.teardowns = make(map[string]*containerTeardown)
|
||||
@@ -363,7 +410,7 @@ func (p *Pool) Release(ctx context.Context, c *Container) {
|
||||
// period above; claiming before the sleep would quietly extend the cap by
|
||||
// GracePeriod. Losing the claim means the reaper already removed this
|
||||
// container, so there is nothing left to stop.
|
||||
if p.claimTeardown(c.ID) {
|
||||
if p.claimRelease(c.ID) {
|
||||
// Detach from ctx before stopping. Release runs on the way out of a
|
||||
// task, and the most important reason a task is on its way out is
|
||||
// that it was cancelled -- at which point ctx is already dead, the
|
||||
|
||||
@@ -560,6 +560,82 @@ func TestReleaseAfterTheReaperFiredDoesNotTearDownTwice(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A max-uptime callback that has already begun cannot be unwound by
|
||||
// Timer.Stop, so it can reach its claim after Release has disarmed the timer
|
||||
// and deleted the teardown entry. Release has torn the container down by then:
|
||||
// the late callback must find nothing to claim, rather than recreating an
|
||||
// unclosed entry behind Release's back and running stop/remove a second time
|
||||
// against a container that is already gone.
|
||||
//
|
||||
// The interleaving is driven by running the reaper's callback body after
|
||||
// Release returned. That is the closest reachable form: nothing in the public
|
||||
// API lets a caller park the callback between the timer firing and its claim --
|
||||
// the window is the callback goroutine's scheduling -- so a test that merely
|
||||
// started a timer and a Release would be a coin flip. This covers the
|
||||
// claim/forget ordering and the Docker calls that follow it; it does not cover
|
||||
// the timer machinery itself.
|
||||
func TestReaperCallbackAfterReleaseDoesNotResurrectTeardown(t *testing.T) {
|
||||
const containerID = "9a8b7c6d5e4f32109a8b7c6d5e4f32109a8b7c6d5e4f32109a8b7c6d5e4f3210"
|
||||
|
||||
var stopCalls, removeCalls atomic.Int32
|
||||
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
||||
writeDockerJSON(t, w, map[string]any{"Id": containerID, "Warnings": []string{}})
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/start"):
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/stop"):
|
||||
stopCalls.Add(1)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID):
|
||||
removeCalls.Add(1)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(dockerAPI.Close)
|
||||
|
||||
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||
|
||||
pool := &Pool{
|
||||
cli: dockerClient,
|
||||
cfg: Config{Image: "test/image", MaxUptime: time.Hour},
|
||||
log: discardLogger(),
|
||||
}
|
||||
|
||||
c, err := pool.Acquire(context.Background(), nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Acquire: %v", err)
|
||||
}
|
||||
pool.Release(context.Background(), c)
|
||||
|
||||
stopAfterRelease, removeAfterRelease := stopCalls.Load(), removeCalls.Load()
|
||||
if stopAfterRelease != 1 || removeAfterRelease != 1 {
|
||||
t.Fatalf("Release stopped the container %d times and removed it %d times, want 1/1: a late callback proves nothing unless the release already did the teardown", stopAfterRelease, removeAfterRelease)
|
||||
}
|
||||
|
||||
pool.reapMaxUptime(context.Background(), c.ID)
|
||||
|
||||
if got := stopCalls.Load(); got != stopAfterRelease {
|
||||
t.Errorf("stop called %d times after Release, want %d: the late reaper tore down an already-released container", got, stopAfterRelease)
|
||||
}
|
||||
if got := removeCalls.Load(); got != removeAfterRelease {
|
||||
t.Errorf("remove called %d times after Release, want %d: the late reaper tore down an already-released container", got, removeAfterRelease)
|
||||
}
|
||||
|
||||
pool.mu.Lock()
|
||||
remaining := len(pool.teardowns)
|
||||
pool.mu.Unlock()
|
||||
if remaining != 0 {
|
||||
t.Errorf("pool holds %d teardown entr(ies) after Release, want 0: the late reaper recreated the entry", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPoolActiveReportsInFlightContainers pins the worker-pool read surface
|
||||
// /status reports as its container line: how many containers this pool
|
||||
// currently holds, and the concurrency cap it enforces.
|
||||
|
||||
Reference in new issue
Block a user