fix(container): cancel a container's max-uptime reaper on release
The timer stayed armed after Release removed the container, so an hour later it stopped and removed an ID Docker had already forgotten and logged two warnings that read like a failed teardown.
This commit is contained in:
1 parent
13d32f34b0
commit
dedafd7d43
2 files changed
+91
-5
No files matched your search
@@ -88,6 +88,9 @@ type Pool struct {
|
||||
|
||||
mu sync.Mutex
|
||||
active int
|
||||
// reapers holds the armed max-uptime timer for each live container,
|
||||
// keyed by ID, so Release can cancel the one it is tearing down.
|
||||
reapers map[string]*time.Timer
|
||||
}
|
||||
|
||||
// Config is the subset of daemon container configuration the pool needs.
|
||||
@@ -238,12 +241,13 @@ func (p *Pool) Acquire(ctx context.Context, mounts []storage.Mount, env []string
|
||||
}
|
||||
|
||||
// armMaxUptime schedules a hard stop and remove for a container once its
|
||||
// lifetime cap elapses. The timer is keyed to the container ID and never
|
||||
// touches p.active: Release (or Close) remains the only owner of the active
|
||||
// slot. Docker stop and remove are idempotent, so a timer that fires after
|
||||
// Release has already removed the container only logs a warning.
|
||||
// lifetime cap elapses. The timer never touches p.active: Release (or Close)
|
||||
// remains the only owner of the active slot. Release cancels the timer, so a
|
||||
// container that finished normally is not reaped a second time against an ID
|
||||
// Docker has already forgotten.
|
||||
func (p *Pool) armMaxUptime(ctx context.Context, id string) {
|
||||
time.AfterFunc(p.cfg.MaxUptime, func() {
|
||||
timer := time.AfterFunc(p.cfg.MaxUptime, func() {
|
||||
p.forgetReaper(id)
|
||||
zero := 0
|
||||
stopCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
@@ -254,6 +258,32 @@ func (p *Pool) armMaxUptime(ctx context.Context, id string) {
|
||||
p.log.Warn("max uptime remove failed", "id", id[:12], "err", err)
|
||||
}
|
||||
})
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.reapers == nil {
|
||||
p.reapers = make(map[string]*time.Timer)
|
||||
}
|
||||
p.reapers[id] = timer
|
||||
}
|
||||
|
||||
// cancelMaxUptime disarms a container's reaper, if it has not already fired.
|
||||
func (p *Pool) cancelMaxUptime(id string) {
|
||||
p.mu.Lock()
|
||||
timer := p.reapers[id]
|
||||
delete(p.reapers, id)
|
||||
p.mu.Unlock()
|
||||
if timer != nil {
|
||||
timer.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// forgetReaper drops a timer's own map entry as it fires, so the map does not
|
||||
// grow for the life of the pool.
|
||||
func (p *Pool) forgetReaper(id string) {
|
||||
p.mu.Lock()
|
||||
delete(p.reapers, id)
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// releaseDecision reports how Release should tear a container down: whether
|
||||
@@ -284,6 +314,7 @@ func (p *Pool) Release(ctx context.Context, c *Container) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
p.cancelMaxUptime(c.ID)
|
||||
honorGrace, stopTimeout := releaseDecision(ctx, p.cfg.GracePeriod)
|
||||
if honorGrace {
|
||||
p.log.Info("container keeping alive for grace period", "id", c.ID[:12], "grace", p.cfg.GracePeriod)
|
||||
|
||||
@@ -367,3 +367,58 @@ func TestAcquireEnforcesMaxUptime(t *testing.T) {
|
||||
t.Fatalf("expected at least one stop and remove, got stop=%d remove=%d", stopCalls.Load(), removeCalls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// A released container is already gone, so its max-uptime reaper has nothing
|
||||
// left to reap. Leaving the timer armed made it fire an hour later against a
|
||||
// dead ID, logging "max uptime stop failed ... No such container" as though
|
||||
// teardown had gone wrong.
|
||||
func TestReleaseCancelsTheMaxUptimeTimer(t *testing.T) {
|
||||
const containerID = "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"
|
||||
|
||||
var stopCalls, removeCalls atomic.Int32
|
||||
dockerAPI := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
||||
writeDockerJSON(t, w, map[string]any{"Id": containerID, "Warnings": []string{}})
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/start"):
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID+"/stop"):
|
||||
stopCalls.Add(1)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
case strings.HasSuffix(r.URL.Path, "/containers/"+containerID):
|
||||
removeCalls.Add(1)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
http.Error(w, "unexpected Docker API path "+r.URL.Path, http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(dockerAPI.Close)
|
||||
|
||||
dockerClient, err := client.New(client.WithHost(dockerAPI.URL), client.WithAPIVersion("1.55"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = dockerClient.Close() })
|
||||
|
||||
pool := &Pool{
|
||||
cli: dockerClient,
|
||||
cfg: Config{Image: "test/image", MaxUptime: 60 * time.Millisecond},
|
||||
log: discardLogger(),
|
||||
}
|
||||
|
||||
c, err := pool.Acquire(context.Background(), nil, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Acquire: %v", err)
|
||||
}
|
||||
pool.Release(context.Background(), c)
|
||||
|
||||
stopAfterRelease, removeAfterRelease := stopCalls.Load(), removeCalls.Load()
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
if got := stopCalls.Load(); got != stopAfterRelease {
|
||||
t.Errorf("stop called %d times after Release, want %d: the max uptime timer still fired", got, stopAfterRelease)
|
||||
}
|
||||
if got := removeCalls.Load(); got != removeAfterRelease {
|
||||
t.Errorf("remove called %d times after Release, want %d: the max uptime timer still fired", got, removeAfterRelease)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user