fix(daemon): park a task whose agent container exits before replying
This commit is contained in:
1 parent
55ad93e8e3
commit
d01a3c4647
3 files changed
+99
-3
No files matched your search
@@ -23,9 +23,15 @@ import (
|
||||
|
||||
// Container wraps a running Docker container.
|
||||
type Container struct {
|
||||
ID string
|
||||
ID string
|
||||
exited <-chan struct{}
|
||||
}
|
||||
|
||||
// Exited closes when the container stops running for any reason: the
|
||||
// max-uptime reaper, an OOM kill, a crash or Release. A Container the pool did
|
||||
// not start returns nil, which never closes.
|
||||
func (c *Container) Exited() <-chan struct{} { return c.exited }
|
||||
|
||||
// TaskPayload is the boot-time brief written to /data/worktree/.git/task.json
|
||||
// before the container starts, per PRD section 3.
|
||||
type TaskPayload struct {
|
||||
@@ -248,7 +254,23 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
||||
}
|
||||
|
||||
p.log.Info("container started", "id", resp.ID[:12], "name", name)
|
||||
return &Container{ID: resp.ID}, nil
|
||||
return &Container{ID: resp.ID, exited: p.watchExit(ctx, resp.ID)}, nil
|
||||
}
|
||||
|
||||
// watchExit returns a channel closed once Docker reports the container is no
|
||||
// longer running. A failed wait says nothing about the container, so it never
|
||||
// closes the channel: a Docker API error must not abort a live run.
|
||||
func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} {
|
||||
exited := make(chan struct{})
|
||||
wait := p.cli.ContainerWait(context.WithoutCancel(ctx), id, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
|
||||
go func() {
|
||||
select {
|
||||
case <-wait.Result:
|
||||
close(exited)
|
||||
case <-wait.Error:
|
||||
}
|
||||
}()
|
||||
return exited
|
||||
}
|
||||
|
||||
// containerTeardown is the one-shot teardown state for a live container: its
|
||||
|
||||
@@ -1276,7 +1276,9 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
|
||||
// worktreerpc clients to that identity, and the daemon registered one
|
||||
// server pair per identity (plus the root pair), so a container-mode
|
||||
// task is always served by its own forge client and worktree manager.
|
||||
d.runViaAgent(ctx, task, repo, profile)
|
||||
runCtx, stopWatch := withContainerExit(ctx, ctr.Exited())
|
||||
d.runViaAgent(runCtx, task, repo, profile)
|
||||
stopWatch()
|
||||
|
||||
// Teardown storage after workflow completes. The Docker backend is a
|
||||
// no-op; future backends (temp volumes, NFS leases) use this hook.
|
||||
@@ -1573,6 +1575,9 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
|
||||
}
|
||||
|
||||
reply, err := d.requestTaskRun(ctx, task.ID, data)
|
||||
if err != nil && errors.Is(context.Cause(ctx), errContainerExited) {
|
||||
err = errContainerExited
|
||||
}
|
||||
if err != nil {
|
||||
d.Log.Error("taskrun request failed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error(), taskstate.ParkTransient)
|
||||
@@ -2075,3 +2080,22 @@ func (d *Daemon) LastPollAt() time.Time {
|
||||
func (d *Daemon) markPoll() {
|
||||
d.lastPollAt.Store(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// errContainerExited is the cancellation cause when a task's agent container
|
||||
// stops before answering its taskrun request.
|
||||
var errContainerExited = errors.New("agent container exited before the task finished")
|
||||
|
||||
// withContainerExit derives a context cancelled when exited closes. A dead
|
||||
// container never answers its taskrun request, so without this the request
|
||||
// blocks forever and the task row stays running.
|
||||
func withContainerExit(ctx context.Context, exited <-chan struct{}) (context.Context, func()) {
|
||||
runCtx, cancel := context.WithCancelCause(ctx)
|
||||
go func() {
|
||||
select {
|
||||
case <-exited:
|
||||
cancel(errContainerExited)
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
}()
|
||||
return runCtx, func() { cancel(nil) }
|
||||
}
|
||||
@@ -2349,3 +2349,53 @@ func TestRunViaAgentCarriesTheWorkflowProfile(t *testing.T) {
|
||||
t.Fatalf("task = %+v, %v; want parked naming the profile", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A container that dies mid-run (max-uptime reaper, OOM, crash) never answers
|
||||
// the taskrun request. The run must end with the task parked, not left
|
||||
// running with the worker blocked on a reply that cannot come.
|
||||
func TestRunViaAgentParksWhenTheContainerExits(t *testing.T) {
|
||||
d, s, busClient := daemonWithNATS(t)
|
||||
ctx := context.Background()
|
||||
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 21, "t", "b", "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := s.ClaimNext(ctx)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("claim: (%v, %v)", task, err)
|
||||
}
|
||||
|
||||
received := make(chan struct{}, 1)
|
||||
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(*natsio.Msg) {
|
||||
received <- struct{}{}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("subscribe: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
exited := make(chan struct{})
|
||||
runCtx, stop := withContainerExit(ctx, exited)
|
||||
defer stop()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
d.runViaAgent(runCtx, task, config.Repo{Owner: "acme", Name: "widget", Base: "main"}, config.AgentProfile{})
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-received:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("archied did not publish a taskrun request")
|
||||
}
|
||||
close(exited)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("runViaAgent still blocked after its container exited")
|
||||
}
|
||||
|
||||
got, err := s.TaskByID(ctx, task.ID)
|
||||
if err != nil || got.Status != workflow.StatusParked || !strings.Contains(got.ParkReason, "agent container exited") {
|
||||
t.Fatalf("task = %+v, %v; want parked naming the container exit", got, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user