fix(daemon): park a task whose agent container exits before replying

This commit is contained in:
sam committed 2026-09-25 08:31:46 +10:00
1 parent 55ad93e8e3
commit d01a3c4647
3 files changed
+99 -3

No files matched your search

+24 -2
View File
@@ -23,9 +23,15 @@ import (
// Container wraps a running Docker container.
type Container struct {
ID string
ID string
exited <-chan struct{}
}
// Exited closes when the container stops running for any reason: the
// max-uptime reaper, an OOM kill, a crash or Release. A Container the pool did
// not start returns nil, which never closes.
func (c *Container) Exited() <-chan struct{} { return c.exited }
// TaskPayload is the boot-time brief written to /data/worktree/.git/task.json
// before the container starts, per PRD section 3.
type TaskPayload struct {
@@ -248,7 +254,23 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
}
p.log.Info("container started", "id", resp.ID[:12], "name", name)
return &Container{ID: resp.ID}, nil
return &Container{ID: resp.ID, exited: p.watchExit(ctx, resp.ID)}, nil
}
// watchExit returns a channel closed once Docker reports the container is no
// longer running. A failed wait says nothing about the container, so it never
// closes the channel: a Docker API error must not abort a live run.
func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} {
exited := make(chan struct{})
wait := p.cli.ContainerWait(context.WithoutCancel(ctx), id, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
go func() {
select {
case <-wait.Result:
close(exited)
case <-wait.Error:
}
}()
return exited
}
// containerTeardown is the one-shot teardown state for a live container: its
+25 -1
View File
@@ -1276,7 +1276,9 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
// worktreerpc clients to that identity, and the daemon registered one
// server pair per identity (plus the root pair), so a container-mode
// task is always served by its own forge client and worktree manager.
d.runViaAgent(ctx, task, repo, profile)
runCtx, stopWatch := withContainerExit(ctx, ctr.Exited())
d.runViaAgent(runCtx, task, repo, profile)
stopWatch()
// Teardown storage after workflow completes. The Docker backend is a
// no-op; future backends (temp volumes, NFS leases) use this hook.
@@ -1573,6 +1575,9 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
}
reply, err := d.requestTaskRun(ctx, task.ID, data)
if err != nil && errors.Is(context.Cause(ctx), errContainerExited) {
err = errContainerExited
}
if err != nil {
d.Log.Error("taskrun request failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error(), taskstate.ParkTransient)
@@ -2075,3 +2080,22 @@ func (d *Daemon) LastPollAt() time.Time {
func (d *Daemon) markPoll() {
d.lastPollAt.Store(time.Now().UnixNano())
}
// errContainerExited is the cancellation cause when a task's agent container
// stops before answering its taskrun request.
var errContainerExited = errors.New("agent container exited before the task finished")
// withContainerExit derives a context cancelled when exited closes. A dead
// container never answers its taskrun request, so without this the request
// blocks forever and the task row stays running.
func withContainerExit(ctx context.Context, exited <-chan struct{}) (context.Context, func()) {
runCtx, cancel := context.WithCancelCause(ctx)
go func() {
select {
case <-exited:
cancel(errContainerExited)
case <-runCtx.Done():
}
}()
return runCtx, func() { cancel(nil) }
}
+50
View File
@@ -2349,3 +2349,53 @@ func TestRunViaAgentCarriesTheWorkflowProfile(t *testing.T) {
t.Fatalf("task = %+v, %v; want parked naming the profile", got, err)
}
}
// A container that dies mid-run (max-uptime reaper, OOM, crash) never answers
// the taskrun request. The run must end with the task parked, not left
// running with the worker blocked on a reply that cannot come.
func TestRunViaAgentParksWhenTheContainerExits(t *testing.T) {
d, s, busClient := daemonWithNATS(t)
ctx := context.Background()
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 21, "t", "b", "", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("claim: (%v, %v)", task, err)
}
received := make(chan struct{}, 1)
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(*natsio.Msg) {
received <- struct{}{}
})
if err != nil {
t.Fatalf("subscribe: %v", err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
exited := make(chan struct{})
runCtx, stop := withContainerExit(ctx, exited)
defer stop()
done := make(chan struct{})
go func() {
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{})
close(done)
}()
select {
case <-received:
case <-time.After(5 * time.Second):
t.Fatal("archied did not publish a taskrun request")
}
close(exited)
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("runViaAgent still blocked after its container exited")
}
got, err := s.TaskByID(ctx, task.ID)
if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, "agent container exited") {
t.Fatalf("task = %+v, %v; want parked naming the container exit", got, err)
}
}