From 07c8bf310db30f2a662009e8490f5388c15d4fc6 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 24 Oct 2026 02:25:25 +1100 Subject: [PATCH] fix(daemon): say how an agent container ended when it dies mid-task --- internal/container/pool.go | 52 ++++++++++++++++++++++++++++++++++--- internal/daemon/daemon.go | 16 +++++++++--- internal/daemon/kit_task.go | 2 +- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/internal/container/pool.go b/internal/container/pool.go index a6c463186..0f595714a 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -15,9 +15,11 @@ import ( "maps" "os" "path/filepath" + "strings" "sync" "time" + "github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/registry" "github.com/moby/moby/client" @@ -29,6 +31,9 @@ import ( type Container struct { ID string exited <-chan struct{} + // detail is written before exited closes, so a reader that saw the close + // may read it without a lock. + detail *string } // Exited closes when the container stops running for any reason: the @@ -36,6 +41,21 @@ type Container struct { // not start returns nil, which never closes. func (c *Container) Exited() <-chan struct{} { return c.exited } +// ExitDetail says how the container ended: its exit code and the last lines it +// wrote. It is empty until Exited closes, and for a container the pool did not +// start. +func (c *Container) ExitDetail() string { + if c.detail == nil { + return "" + } + select { + case <-c.exited: + return *c.detail + default: + return "" + } +} + // TaskPayload is the boot-time brief written to /data/worktree/.git/task.json // before the container starts. type TaskPayload struct { @@ -340,7 +360,8 @@ func (p *Pool) started(ctx context.Context, maxUptime time.Duration, id, name st p.armMaxUptime(ctx, maxUptime, id) } p.log.Info("container started", "id", id[:12], "name", name) - return &Container{ID: id, exited: p.watchExit(ctx, id)} + detail := new(string) + return &Container{ID: id, exited: p.watchExit(ctx, id, detail), detail: detail} } // Client is the pool's Docker client, for callers that provision what a @@ -350,12 +371,13 @@ func (p *Pool) Client() *client.Client { return p.cli } // watchExit returns a channel closed once Docker reports the container is no // longer running. A failed wait says nothing about the container, so it never // closes the channel: a Docker API error must not abort a live run. -func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} { +func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan struct{} { exited := make(chan struct{}) wait := p.cli.ContainerWait(context.WithoutCancel(ctx), id, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning}) go func() { select { - case <-wait.Result: + case result := <-wait.Result: + *detail = p.exitDetail(context.WithoutCancel(ctx), id, result.StatusCode) close(exited) case <-wait.Error: } @@ -363,6 +385,30 @@ func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} { return exited } +// exitDetail describes a finished container for the operator. The container is +// auto-removed, so its output is read best effort: a lost race leaves the code. +func (p *Pool) exitDetail(ctx context.Context, id string, code int64) string { + detail := fmt.Sprintf("exit code %d", code) + if code == 137 { + detail += " (killed; often out of memory)" + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + logs, err := p.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "5"}) + if err != nil { + return detail + } + defer logs.Close() + var out strings.Builder + if _, err := stdcopy.StdCopy(&out, &out, io.LimitReader(logs, 4096)); err != nil && out.Len() == 0 { + return detail + } + if tail := strings.TrimSpace(out.String()); tail != "" { + detail += "; last output: " + strings.Join(strings.Fields(strings.ReplaceAll(tail, "\n", " | ")), " ") + } + return detail +} + // containerTeardown is a live container's max-uptime timer and whether its // Docker teardown has been claimed. type containerTeardown struct { diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index d55d669a5..7052e0e3d 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -1527,7 +1527,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol // Run the whole task in archie-agent, which calls back to the daemon for // store, forge and push operations. limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std()) - runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited()) + runCtx, stopWatch := withContainerExit(limitCtx, ctr) d.runViaAgent(runCtx, task, repo, profile, nil, credential) stopWatch() stopLimit() @@ -2495,14 +2495,24 @@ func (d *Daemon) markPoll(loop string) { // stops before answering its taskrun request. var errContainerExited = errors.New("agent container exited before the task finished") +// exitWatcher is a running agent container, seen only for how it ends. +type exitWatcher interface { + Exited() <-chan struct{} + ExitDetail() string +} + // withContainerExit derives a context cancelled when exited closes. A dead // container never answers its taskrun request, so without this the request // blocks forever and the task row stays running. -func withContainerExit(ctx context.Context, exited <-chan struct{}) (context.Context, func()) { +func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) { runCtx, cancel := context.WithCancelCause(ctx) go func() { select { - case <-exited: + case <-ctr.Exited(): + if detail := ctr.ExitDetail(); detail != "" { + cancel(fmt.Errorf("%w: %s", errContainerExited, detail)) + return + } cancel(errContainerExited) case <-runCtx.Done(): } diff --git a/internal/daemon/kit_task.go b/internal/daemon/kit_task.go index faedbcc95..2b8a563d2 100644 --- a/internal/daemon/kit_task.go +++ b/internal/daemon/kit_task.go @@ -86,7 +86,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi } limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std()) - runCtx, stopWatch := withContainerExit(limitCtx, run.Container.Exited()) + runCtx, stopWatch := withContainerExit(limitCtx, run.Container) d.runViaAgent(runCtx, task, repo, profile, &run.Harness, credential) stopWatch() stopLimit()