From 084fa71e06c781137dd3d24921d3500744728a12 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 25 Sep 2026 08:31:46 +1000 Subject: [PATCH] fix(daemon): park a task whose agent container exits before replying --- internal/container/pool.go | 26 ++++++++++++++++-- internal/daemon/daemon.go | 26 +++++++++++++++++- internal/daemon/daemon_test.go | 50 ++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/internal/container/pool.go b/internal/container/pool.go index fa4c76da1..cd6b9489c 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -23,9 +23,15 @@ import ( // Container wraps a running Docker container. type Container struct { - ID string + ID string + exited <-chan struct{} } +// Exited closes when the container stops running for any reason: the +// max-uptime reaper, an OOM kill, a crash or Release. A Container the pool did +// not start returns nil, which never closes. +func (c *Container) Exited() <-chan struct{} { return c.exited } + // TaskPayload is the boot-time brief written to /data/worktree/.git/task.json // before the container starts, per PRD section 3. type TaskPayload struct { @@ -248,7 +254,23 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount } p.log.Info("container started", "id", resp.ID[:12], "name", name) - return &Container{ID: resp.ID}, nil + return &Container{ID: resp.ID, exited: p.watchExit(ctx, resp.ID)}, nil +} + +// watchExit returns a channel closed once Docker reports the container is no +// longer running. A failed wait says nothing about the container, so it never +// closes the channel: a Docker API error must not abort a live run. +func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} { + exited := make(chan struct{}) + wait := p.cli.ContainerWait(context.WithoutCancel(ctx), id, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}) + go func() { + select { + case <-wait.Result: + close(exited) + case <-wait.Error: + } + }() + return exited } // containerTeardown is the one-shot teardown state for a live container: its diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 16ffafa9d..63b30df40 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -1276,7 +1276,9 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) { // worktreerpc clients to that identity, and the daemon registered one // server pair per identity (plus the root pair), so a container-mode // task is always served by its own forge client and worktree manager. - d.runViaAgent(ctx, task, repo, profile) + runCtx, stopWatch := withContainerExit(ctx, ctr.Exited()) + d.runViaAgent(runCtx, task, repo, profile) + stopWatch() // Teardown storage after workflow completes. The Docker backend is a // no-op; future backends (temp volumes, NFS leases) use this hook. @@ -1573,6 +1575,9 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf } reply, err := d.requestTaskRun(ctx, task.ID, data) + if err != nil && errors.Is(context.Cause(ctx), errContainerExited) { + err = errContainerExited + } if err != nil { d.Log.Error("taskrun request failed", "task", task.ID, "err", err) d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error(), taskstate.ParkTransient) @@ -2075,3 +2080,22 @@ func (d *Daemon) LastPollAt() time.Time { func (d *Daemon) markPoll() { d.lastPollAt.Store(time.Now().UnixNano()) } + +// errContainerExited is the cancellation cause when a task's agent container +// stops before answering its taskrun request. +var errContainerExited = errors.New("agent container exited before the task finished") + +// withContainerExit derives a context cancelled when exited closes. A dead +// container never answers its taskrun request, so without this the request +// blocks forever and the task row stays running. +func withContainerExit(ctx context.Context, exited <-chan struct{}) (context.Context, func()) { + runCtx, cancel := context.WithCancelCause(ctx) + go func() { + select { + case <-exited: + cancel(errContainerExited) + case <-runCtx.Done(): + } + }() + return runCtx, func() { cancel(nil) } +} diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 7546ab4e2..d9e850e72 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -2349,3 +2349,53 @@ func TestRunViaAgentCarriesTheWorkflowProfile(t *testing.T) { t.Fatalf("task = %+v, %v; want parked naming the profile", got, err) } } + +// A container that dies mid-run (max-uptime reaper, OOM, crash) never answers +// the taskrun request. The run must end with the task parked, not left +// running with the worker blocked on a reply that cannot come. +func TestRunViaAgentParksWhenTheContainerExits(t *testing.T) { + d, s, busClient := daemonWithNATS(t) + ctx := context.Background() + if _, err := s.EnqueueIssue(ctx, "acme", "widget", 21, "t", "b", "", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("claim: (%v, %v)", task, err) + } + + received := make(chan struct{}, 1) + sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(*natsio.Msg) { + received <- struct{}{} + }) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + + exited := make(chan struct{}) + runCtx, stop := withContainerExit(ctx, exited) + defer stop() + done := make(chan struct{}) + go func() { + d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{}) + close(done) + }() + + select { + case <-received: + case <-time.After(5 * time.Second): + t.Fatal("archied did not publish a taskrun request") + } + close(exited) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("runViaAgent still blocked after its container exited") + } + + got, err := s.TaskByID(ctx, task.ID) + if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, "agent container exited") { + t.Fatalf("task = %+v, %v; want parked naming the container exit", got, err) + } +}