fix(daemon): say how an agent container ended when it dies mid-task

This commit is contained in:
sam committed 2026-10-24 02:25:25 +11:00
1 parent dc6c4826bc
commit 07c8bf310d
3 files changed
+63 -7

No files matched your search

+49 -3
View File
@@ -15,9 +15,11 @@ import (
"maps"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/moby/moby/api/pkg/stdcopy"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/registry"
"github.com/moby/moby/client"
@@ -29,6 +31,9 @@ import (
type Container struct {
ID string
exited <-chan struct{}
// detail is written before exited closes, so a reader that saw the close
// may read it without a lock.
detail *string
}
// Exited closes when the container stops running for any reason: the
@@ -36,6 +41,21 @@ type Container struct {
// not start returns nil, which never closes.
func (c *Container) Exited() <-chan struct{} { return c.exited }
// ExitDetail says how the container ended: its exit code and the last lines it
// wrote. It is empty until Exited closes, and for a container the pool did not
// start.
func (c *Container) ExitDetail() string {
if c.detail == nil {
return ""
}
select {
case <-c.exited:
return *c.detail
default:
return ""
}
}
// TaskPayload is the boot-time brief written to /data/worktree/.git/task.json
// before the container starts.
type TaskPayload struct {
@@ -340,7 +360,8 @@ func (p *Pool) started(ctx context.Context, maxUptime time.Duration, id, name st
p.armMaxUptime(ctx, maxUptime, id)
}
p.log.Info("container started", "id", id[:12], "name", name)
return &Container{ID: id, exited: p.watchExit(ctx, id)}
detail := new(string)
return &Container{ID: id, exited: p.watchExit(ctx, id, detail), detail: detail}
}
// Client is the pool's Docker client, for callers that provision what a
@@ -350,12 +371,13 @@ func (p *Pool) Client() *client.Client { return p.cli }
// watchExit returns a channel closed once Docker reports the container is no
// longer running. A failed wait says nothing about the container, so it never
// closes the channel: a Docker API error must not abort a live run.
func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} {
func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan struct{} {
exited := make(chan struct{})
wait := p.cli.ContainerWait(context.WithoutCancel(ctx), id, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
go func() {
select {
case <-wait.Result:
case result := <-wait.Result:
*detail = p.exitDetail(context.WithoutCancel(ctx), id, result.StatusCode)
close(exited)
case <-wait.Error:
}
@@ -363,6 +385,30 @@ func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} {
return exited
}
// exitDetail describes a finished container for the operator. The container is
// auto-removed, so its output is read best effort: a lost race leaves the code.
func (p *Pool) exitDetail(ctx context.Context, id string, code int64) string {
detail := fmt.Sprintf("exit code %d", code)
if code == 137 {
detail += " (killed; often out of memory)"
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
logs, err := p.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "5"})
if err != nil {
return detail
}
defer logs.Close()
var out strings.Builder
if _, err := stdcopy.StdCopy(&out, &out, io.LimitReader(logs, 4096)); err != nil && out.Len() == 0 {
return detail
}
if tail := strings.TrimSpace(out.String()); tail != "" {
detail += "; last output: " + strings.Join(strings.Fields(strings.ReplaceAll(tail, "\n", " | ")), " ")
}
return detail
}
// containerTeardown is a live container's max-uptime timer and whether its
// Docker teardown has been claimed.
type containerTeardown struct {
+13 -3
View File
@@ -1527,7 +1527,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol
// Run the whole task in archie-agent, which calls back to the daemon for
// store, forge and push operations.
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited())
runCtx, stopWatch := withContainerExit(limitCtx, ctr)
d.runViaAgent(runCtx, task, repo, profile, nil, credential)
stopWatch()
stopLimit()
@@ -2495,14 +2495,24 @@ func (d *Daemon) markPoll(loop string) {
// stops before answering its taskrun request.
var errContainerExited = errors.New("agent container exited before the task finished")
// exitWatcher is a running agent container, seen only for how it ends.
type exitWatcher interface {
Exited() <-chan struct{}
ExitDetail() string
}
// withContainerExit derives a context cancelled when exited closes. A dead
// container never answers its taskrun request, so without this the request
// blocks forever and the task row stays running.
func withContainerExit(ctx context.Context, exited <-chan struct{}) (context.Context, func()) {
func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) {
runCtx, cancel := context.WithCancelCause(ctx)
go func() {
select {
case <-exited:
case <-ctr.Exited():
if detail := ctr.ExitDetail(); detail != "" {
cancel(fmt.Errorf("%w: %s", errContainerExited, detail))
return
}
cancel(errContainerExited)
case <-runCtx.Done():
}
+1 -1
View File
@@ -86,7 +86,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi
}
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
runCtx, stopWatch := withContainerExit(limitCtx, run.Container.Exited())
runCtx, stopWatch := withContainerExit(limitCtx, run.Container)
d.runViaAgent(runCtx, task, repo, profile, &run.Harness, credential)
stopWatch()
stopLimit()