diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index c80d25d6f..58c5b12c3 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -269,13 +269,13 @@ func parseListenAddr(addr, defaultHost string, defaultPort int) (string, int) { } // subscribeSystemLogs subscribes to every task's system log subject at once. -// A sandboxed container's own stderr disappears at AutoRemove; archie-agent -// ships it here instead (agentexec.SubjectForSystem), and this is where it -// lands: one wildcard subscription demuxed by taskID and routed through -// taskLogs, which is opened for a task's duration in Daemon.process, so a -// message for a task not currently open there -- late, duplicate, or from -// an attempt this instance never dispatched -- is expected and silently -// dropped rather than treated as an error. +// archie-agent's own log lines ship here (agentexec.SubjectForSystem) rather +// than staying on its stderr, and this is where they land: one wildcard +// subscription demuxed by taskID and routed through taskLogs, which is opened +// for a task's duration in Daemon.process, so a message for a task not +// currently open there -- late, duplicate, or from an attempt this instance +// never dispatched -- is expected and silently dropped rather than treated as +// an error. func subscribeSystemLogs(nc *natsio.Conn, taskLogs *logging.TaskRegistry, log *slog.Logger) (unsubscribe func(), err error) { sub, err := nc.Subscribe(agentexec.SubjectSystemWildcard, func(msg *natsio.Msg) { taskID, ok := agentexec.TaskIDFromSystemSubject(msg.Subject) diff --git a/internal/container/pool.go b/internal/container/pool.go index 4f380dae1..2518be696 100644 --- a/internal/container/pool.go +++ b/internal/container/pool.go @@ -5,6 +5,7 @@ package container import ( + "bytes" "context" "encoding/base64" "encoding/json" @@ -15,9 +16,12 @@ import ( "maps" "os" "path/filepath" + "strconv" + "strings" "sync" "time" + "github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/registry" "github.com/moby/moby/client" @@ -39,8 +43,9 @@ type Container struct { // not start returns nil, which never closes. func (c *Container) Exited() <-chan struct{} { return c.exited } -// ExitDetail says how the container ended: its exit code. It is empty until -// Exited closes, and for a container the pool did not start. +// ExitDetail says how the container ended: its exit code, and the runtime's +// crash line when it panicked. It is empty until Exited closes, and for a +// container the pool did not start. func (c *Container) ExitDetail() string { if c.detail == nil { return "" @@ -260,7 +265,10 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount name := fmt.Sprintf("%s-agent-%d", cfg.Instance, time.Now().UnixNano()) - hostConfig := &container.HostConfig{AutoRemove: true} + // Release owns removal rather than Docker's auto-remove: a crash writes its + // panic trace to stderr, and the container's log is unreadable the moment + // Docker removes it. + hostConfig := &container.HostConfig{} for i, m := range storage.ConvertMounts(mounts) { if string(m.Type) == storage.MountTypeBind { label := "Z" @@ -374,7 +382,7 @@ func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan go func() { select { case result := <-wait.Result: - *detail = exitDetail(result.StatusCode) + *detail = p.exitDetail(ctx, id, result.StatusCode) close(exited) case <-wait.Error: } @@ -382,9 +390,80 @@ func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan return exited } -// exitDetail describes a finished container for the operator. Its output is -// left out: the reason reaches the dashboard, logs and notifications, and an -// agent's last lines can carry a credential. +// exitDetail describes a finished container for the operator. The detail +// becomes the task's park reason, which reaches the dashboard and chat, and an +// agent's output can carry a credential, so it takes only the crash line. The +// full stderr tail, the only account of a Go panic, goes to the daemon log. +func (p *Pool) exitDetail(ctx context.Context, id string, code int64) string { + detail := exitDetail(code) + if code == 0 { + return detail + } + tail := p.stderrTail(ctx, id) + if tail == "" { + return detail + } + p.log.ErrorContext(ctx, "agent container crashed", "id", id[:12], "code", code, "stderr", tail) + if line := crashLine(tail); line != "" { + detail += ": " + line + } + return detail +} + +// crashLineBytes bounds the crash line carried into the park reason. +const crashLineBytes = 200 + +// crashLine picks the runtime's own summary of a crash from a stderr tail. +func crashLine(tail string) string { + for line := range strings.Lines(tail) { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "panic: ") || strings.HasPrefix(line, "fatal error: ") { + if len(line) > crashLineBytes { + line = line[:crashLineBytes] + "…" + } + return line + } + } + return "" +} + +const ( + // crashTailLines bounds the stderr tail read after a crash: enough for a + // panic's opening line and the frames around the fault. + crashTailLines = 100 + // crashTailBytes bounds the same tail by size, for a container that writes + // one enormous line or a multi-megabyte stack dump. + crashTailBytes = 8 << 10 +) + +// stderrTail returns what the container last wrote to stderr. It reads the log +// of a stopped container, which exists until Release removes it. Best effort: a +// container with no output, or a Docker error, yields "". +func (p *Pool) stderrTail(ctx context.Context, id string) string { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + reader, err := p.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{ + ShowStderr: true, + Tail: strconv.Itoa(crashTailLines), + }) + if err != nil { + p.log.WarnContext(ctx, "container stderr unavailable after exit", "id", id[:12], "err", err) + return "" + } + defer func() { + if err := reader.Close(); err != nil { + p.log.WarnContext(ctx, "container log reader close failed", "id", id[:12], "err", err) + } + }() + var stderr bytes.Buffer + // stdout is not part of the detail. Hitting the byte cap cuts the stream + // short, so the partial text is kept rather than the read reported as a + // failure. + _, _ = stdcopy.StdCopy(io.Discard, &stderr, io.LimitReader(reader, crashTailBytes)) + return strings.TrimSpace(stderr.String()) +} + +// exitDetail is the operator-facing one-line cause of a container's exit. func exitDetail(code int64) string { if code == 137 { return "exit code 137 (killed; often out of memory)" diff --git a/internal/container/pool_test.go b/internal/container/pool_test.go index 613d494c5..48a85b20e 100644 --- a/internal/container/pool_test.go +++ b/internal/container/pool_test.go @@ -1,7 +1,10 @@ package container import ( + "encoding/binary" "encoding/json" + "fmt" + "io" "log/slog" "net/http" "net/http/httptest" @@ -9,6 +12,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/moby/moby/client" ) @@ -74,3 +78,91 @@ func TestPoolReapsOnlyOwnInstance(t *testing.T) { }) } } + +// TestExitDetailCarriesCrashStderr runs the exit watch against a fake Engine: a +// container that fails writes its panic trace to stderr, and the panic line has +// to reach the exit detail before Release removes the container, without the +// rest of its output. +func TestExitDetailCarriesCrashStderr(t *testing.T) { + tests := []struct { + name string + code int64 + stderr []string + want []string + reject []string + wantLogs bool + }{ + { + name: "crash keeps only the panic line", + code: 2, + stderr: []string{"token=s3cret", "panic: runtime error: boom", "goroutine 1 [running]:"}, + want: []string{"exit code 2", "panic: runtime error: boom"}, + reject: []string{"s3cret", "goroutine"}, + wantLogs: true, + }, + { + name: "clean exit asks for no log", + code: 0, + stderr: []string{"finished"}, + want: []string{"exit code 0"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var logsRequested bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/wait"): + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"StatusCode":%d}`, tt.code) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/logs"): + logsRequested = true + for _, line := range tt.stderr { + writeStderrFrame(t, w, line) + } + default: + http.NotFound(w, r) + } + })) + defer server.Close() + cli, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.54")) + if err != nil { + t.Fatal(err) + } + defer cli.Close() + p := &Pool{cli: cli, log: slog.New(slog.DiscardHandler)} + detail := new(string) + exited := p.watchExit(t.Context(), "crash00000000", detail) + select { + case <-exited: + case <-time.After(5 * time.Second): + t.Fatal("exit watch did not report the container's exit") + } + for _, want := range tt.want { + if !strings.Contains(*detail, want) { + t.Fatalf("exit detail %q does not contain %q", *detail, want) + } + } + for _, reject := range tt.reject { + if strings.Contains(*detail, reject) { + t.Fatalf("exit detail %q carries %q, which belongs only in the daemon log", *detail, reject) + } + } + if logsRequested != tt.wantLogs { + t.Fatalf("log requested = %v, want %v", logsRequested, tt.wantLogs) + } + }) + } +} + +// writeStderrFrame writes one line as a Docker multiplexed stderr frame, the +// shape ContainerLogs returns for a container without a TTY. +func writeStderrFrame(t *testing.T, w io.Writer, line string) { + t.Helper() + payload := []byte(line + "\n") + frame := [8]byte{2} + binary.BigEndian.PutUint32(frame[4:], uint32(len(payload))) + if _, err := w.Write(append(frame[:], payload...)); err != nil { + t.Error(err) + } +}