fix(daemon): say how an agent container ended when it dies mid-task
This commit is contained in:
1 parent
dc6c4826bc
commit
07c8bf310d
3 files changed
+63
-7
No files matched your search
@@ -15,9 +15,11 @@ import (
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/registry"
|
||||
"github.com/moby/moby/client"
|
||||
@@ -29,6 +31,9 @@ import (
|
||||
type Container struct {
|
||||
ID string
|
||||
exited <-chan struct{}
|
||||
// detail is written before exited closes, so a reader that saw the close
|
||||
// may read it without a lock.
|
||||
detail *string
|
||||
}
|
||||
|
||||
// Exited closes when the container stops running for any reason: the
|
||||
@@ -36,6 +41,21 @@ type Container struct {
|
||||
// not start returns nil, which never closes.
|
||||
func (c *Container) Exited() <-chan struct{} { return c.exited }
|
||||
|
||||
// ExitDetail says how the container ended: its exit code and the last lines it
|
||||
// wrote. It is empty until Exited closes, and for a container the pool did not
|
||||
// start.
|
||||
func (c *Container) ExitDetail() string {
|
||||
if c.detail == nil {
|
||||
return ""
|
||||
}
|
||||
select {
|
||||
case <-c.exited:
|
||||
return *c.detail
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// TaskPayload is the boot-time brief written to /data/worktree/.git/task.json
|
||||
// before the container starts.
|
||||
type TaskPayload struct {
|
||||
@@ -340,7 +360,8 @@ func (p *Pool) started(ctx context.Context, maxUptime time.Duration, id, name st
|
||||
p.armMaxUptime(ctx, maxUptime, id)
|
||||
}
|
||||
p.log.Info("container started", "id", id[:12], "name", name)
|
||||
return &Container{ID: id, exited: p.watchExit(ctx, id)}
|
||||
detail := new(string)
|
||||
return &Container{ID: id, exited: p.watchExit(ctx, id, detail), detail: detail}
|
||||
}
|
||||
|
||||
// Client is the pool's Docker client, for callers that provision what a
|
||||
@@ -350,12 +371,13 @@ func (p *Pool) Client() *client.Client { return p.cli }
|
||||
// watchExit returns a channel closed once Docker reports the container is no
|
||||
// longer running. A failed wait says nothing about the container, so it never
|
||||
// closes the channel: a Docker API error must not abort a live run.
|
||||
func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} {
|
||||
func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan struct{} {
|
||||
exited := make(chan struct{})
|
||||
wait := p.cli.ContainerWait(context.WithoutCancel(ctx), id, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
|
||||
go func() {
|
||||
select {
|
||||
case <-wait.Result:
|
||||
case result := <-wait.Result:
|
||||
*detail = p.exitDetail(context.WithoutCancel(ctx), id, result.StatusCode)
|
||||
close(exited)
|
||||
case <-wait.Error:
|
||||
}
|
||||
@@ -363,6 +385,30 @@ func (p *Pool) watchExit(ctx context.Context, id string) <-chan struct{} {
|
||||
return exited
|
||||
}
|
||||
|
||||
// exitDetail describes a finished container for the operator. The container is
|
||||
// auto-removed, so its output is read best effort: a lost race leaves the code.
|
||||
func (p *Pool) exitDetail(ctx context.Context, id string, code int64) string {
|
||||
detail := fmt.Sprintf("exit code %d", code)
|
||||
if code == 137 {
|
||||
detail += " (killed; often out of memory)"
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
logs, err := p.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Tail: "5"})
|
||||
if err != nil {
|
||||
return detail
|
||||
}
|
||||
defer logs.Close()
|
||||
var out strings.Builder
|
||||
if _, err := stdcopy.StdCopy(&out, &out, io.LimitReader(logs, 4096)); err != nil && out.Len() == 0 {
|
||||
return detail
|
||||
}
|
||||
if tail := strings.TrimSpace(out.String()); tail != "" {
|
||||
detail += "; last output: " + strings.Join(strings.Fields(strings.ReplaceAll(tail, "\n", " | ")), " ")
|
||||
}
|
||||
return detail
|
||||
}
|
||||
|
||||
// containerTeardown is a live container's max-uptime timer and whether its
|
||||
// Docker teardown has been claimed.
|
||||
type containerTeardown struct {
|
||||
|
||||
@@ -1527,7 +1527,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol
|
||||
// Run the whole task in archie-agent, which calls back to the daemon for
|
||||
// store, forge and push operations.
|
||||
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
||||
runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited())
|
||||
runCtx, stopWatch := withContainerExit(limitCtx, ctr)
|
||||
d.runViaAgent(runCtx, task, repo, profile, nil, credential)
|
||||
stopWatch()
|
||||
stopLimit()
|
||||
@@ -2495,14 +2495,24 @@ func (d *Daemon) markPoll(loop string) {
|
||||
// stops before answering its taskrun request.
|
||||
var errContainerExited = errors.New("agent container exited before the task finished")
|
||||
|
||||
// exitWatcher is a running agent container, seen only for how it ends.
|
||||
type exitWatcher interface {
|
||||
Exited() <-chan struct{}
|
||||
ExitDetail() string
|
||||
}
|
||||
|
||||
// withContainerExit derives a context cancelled when exited closes. A dead
|
||||
// container never answers its taskrun request, so without this the request
|
||||
// blocks forever and the task row stays running.
|
||||
func withContainerExit(ctx context.Context, exited <-chan struct{}) (context.Context, func()) {
|
||||
func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) {
|
||||
runCtx, cancel := context.WithCancelCause(ctx)
|
||||
go func() {
|
||||
select {
|
||||
case <-exited:
|
||||
case <-ctr.Exited():
|
||||
if detail := ctr.ExitDetail(); detail != "" {
|
||||
cancel(fmt.Errorf("%w: %s", errContainerExited, detail))
|
||||
return
|
||||
}
|
||||
cancel(errContainerExited)
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ func (d *Daemon) runKitTask(ctx context.Context, task *workflow.Task, repo confi
|
||||
}
|
||||
|
||||
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
||||
runCtx, stopWatch := withContainerExit(limitCtx, run.Container.Exited())
|
||||
runCtx, stopWatch := withContainerExit(limitCtx, run.Container)
|
||||
d.runViaAgent(runCtx, task, repo, profile, &run.Harness, credential)
|
||||
stopWatch()
|
||||
stopLimit()
|
||||
|
||||
Reference in new issue
Block a user