fix(container): report a crashed container's panic line and log its stderr

This commit is contained in:
sam committed 2026-10-26 18:24:46 +11:00
1 parent 06d08f422b
commit 877d45f275
3 files changed
+185 -14

No files matched your search

+7 -7
View File
@@ -269,13 +269,13 @@ func parseListenAddr(addr, defaultHost string, defaultPort int) (string, int) {
}
// subscribeSystemLogs subscribes to every task's system log subject at once.
// A sandboxed container's own stderr disappears at AutoRemove; archie-agent
// ships it here instead (agentexec.SubjectForSystem), and this is where it
// lands: one wildcard subscription demuxed by taskID and routed through
// taskLogs, which is opened for a task's duration in Daemon.process, so a
// message for a task not currently open there -- late, duplicate, or from
// an attempt this instance never dispatched -- is expected and silently
// dropped rather than treated as an error.
// archie-agent's own log lines ship here (agentexec.SubjectForSystem) rather
// than staying on its stderr, and this is where they land: one wildcard
// subscription demuxed by taskID and routed through taskLogs, which is opened
// for a task's duration in Daemon.process, so a message for a task not
// currently open there -- late, duplicate, or from an attempt this instance
// never dispatched -- is expected and silently dropped rather than treated as
// an error.
func subscribeSystemLogs(nc *natsio.Conn, taskLogs *logging.TaskRegistry, log *slog.Logger) (unsubscribe func(), err error) {
sub, err := nc.Subscribe(agentexec.SubjectSystemWildcard, func(msg *natsio.Msg) {
taskID, ok := agentexec.TaskIDFromSystemSubject(msg.Subject)
+86 -7
View File
@@ -5,6 +5,7 @@
package container
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
@@ -15,9 +16,12 @@ import (
"maps"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/moby/moby/api/pkg/stdcopy"
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/registry"
"github.com/moby/moby/client"
@@ -39,8 +43,9 @@ type Container struct {
// not start returns nil, which never closes.
func (c *Container) Exited() <-chan struct{} { return c.exited }
// ExitDetail says how the container ended: its exit code. It is empty until
// Exited closes, and for a container the pool did not start.
// ExitDetail says how the container ended: its exit code, and the runtime's
// crash line when it panicked. It is empty until Exited closes, and for a
// container the pool did not start.
func (c *Container) ExitDetail() string {
if c.detail == nil {
return ""
@@ -260,7 +265,10 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
name := fmt.Sprintf("%s-agent-%d", cfg.Instance, time.Now().UnixNano())
hostConfig := &container.HostConfig{AutoRemove: true}
// Release owns removal rather than Docker's auto-remove: a crash writes its
// panic trace to stderr, and the container's log is unreadable the moment
// Docker removes it.
hostConfig := &container.HostConfig{}
for i, m := range storage.ConvertMounts(mounts) {
if string(m.Type) == storage.MountTypeBind {
label := "Z"
@@ -374,7 +382,7 @@ func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan
go func() {
select {
case result := <-wait.Result:
*detail = exitDetail(result.StatusCode)
*detail = p.exitDetail(ctx, id, result.StatusCode)
close(exited)
case <-wait.Error:
}
@@ -382,9 +390,80 @@ func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan
return exited
}
// exitDetail describes a finished container for the operator. Its output is
// left out: the reason reaches the dashboard, logs and notifications, and an
// agent's last lines can carry a credential.
// exitDetail describes a finished container for the operator. The detail
// becomes the task's park reason, which reaches the dashboard and chat, and an
// agent's output can carry a credential, so it takes only the crash line. The
// full stderr tail, the only account of a Go panic, goes to the daemon log.
func (p *Pool) exitDetail(ctx context.Context, id string, code int64) string {
detail := exitDetail(code)
if code == 0 {
return detail
}
tail := p.stderrTail(ctx, id)
if tail == "" {
return detail
}
p.log.ErrorContext(ctx, "agent container crashed", "id", id[:12], "code", code, "stderr", tail)
if line := crashLine(tail); line != "" {
detail += ": " + line
}
return detail
}
// crashLineBytes bounds the crash line carried into the park reason.
const crashLineBytes = 200
// crashLine picks the runtime's own summary of a crash from a stderr tail.
func crashLine(tail string) string {
for line := range strings.Lines(tail) {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "panic: ") || strings.HasPrefix(line, "fatal error: ") {
if len(line) > crashLineBytes {
line = line[:crashLineBytes] + "…"
}
return line
}
}
return ""
}
const (
// crashTailLines bounds the stderr tail read after a crash: enough for a
// panic's opening line and the frames around the fault.
crashTailLines = 100
// crashTailBytes bounds the same tail by size, for a container that writes
// one enormous line or a multi-megabyte stack dump.
crashTailBytes = 8 << 10
)
// stderrTail returns what the container last wrote to stderr. It reads the log
// of a stopped container, which exists until Release removes it. Best effort: a
// container with no output, or a Docker error, yields "".
func (p *Pool) stderrTail(ctx context.Context, id string) string {
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
reader, err := p.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{
ShowStderr: true,
Tail: strconv.Itoa(crashTailLines),
})
if err != nil {
p.log.WarnContext(ctx, "container stderr unavailable after exit", "id", id[:12], "err", err)
return ""
}
defer func() {
if err := reader.Close(); err != nil {
p.log.WarnContext(ctx, "container log reader close failed", "id", id[:12], "err", err)
}
}()
var stderr bytes.Buffer
// stdout is not part of the detail. Hitting the byte cap cuts the stream
// short, so the partial text is kept rather than the read reported as a
// failure.
_, _ = stdcopy.StdCopy(io.Discard, &stderr, io.LimitReader(reader, crashTailBytes))
return strings.TrimSpace(stderr.String())
}
// exitDetail is the operator-facing one-line cause of a container's exit.
func exitDetail(code int64) string {
if code == 137 {
return "exit code 137 (killed; often out of memory)"
+92
View File
@@ -1,7 +1,10 @@
package container
import (
"encoding/binary"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
@@ -9,6 +12,7 @@ import (
"strings"
"sync"
"testing"
"time"
"github.com/moby/moby/client"
)
@@ -74,3 +78,91 @@ func TestPoolReapsOnlyOwnInstance(t *testing.T) {
})
}
}
// TestExitDetailCarriesCrashStderr runs the exit watch against a fake Engine: a
// container that fails writes its panic trace to stderr, and the panic line has
// to reach the exit detail before Release removes the container, without the
// rest of its output.
func TestExitDetailCarriesCrashStderr(t *testing.T) {
tests := []struct {
name string
code int64
stderr []string
want []string
reject []string
wantLogs bool
}{
{
name: "crash keeps only the panic line",
code: 2,
stderr: []string{"token=s3cret", "panic: runtime error: boom", "goroutine 1 [running]:"},
want: []string{"exit code 2", "panic: runtime error: boom"},
reject: []string{"s3cret", "goroutine"},
wantLogs: true,
},
{
name: "clean exit asks for no log",
code: 0,
stderr: []string{"finished"},
want: []string{"exit code 0"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var logsRequested bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/wait"):
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"StatusCode":%d}`, tt.code)
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/logs"):
logsRequested = true
for _, line := range tt.stderr {
writeStderrFrame(t, w, line)
}
default:
http.NotFound(w, r)
}
}))
defer server.Close()
cli, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.54"))
if err != nil {
t.Fatal(err)
}
defer cli.Close()
p := &Pool{cli: cli, log: slog.New(slog.DiscardHandler)}
detail := new(string)
exited := p.watchExit(t.Context(), "crash00000000", detail)
select {
case <-exited:
case <-time.After(5 * time.Second):
t.Fatal("exit watch did not report the container's exit")
}
for _, want := range tt.want {
if !strings.Contains(*detail, want) {
t.Fatalf("exit detail %q does not contain %q", *detail, want)
}
}
for _, reject := range tt.reject {
if strings.Contains(*detail, reject) {
t.Fatalf("exit detail %q carries %q, which belongs only in the daemon log", *detail, reject)
}
}
if logsRequested != tt.wantLogs {
t.Fatalf("log requested = %v, want %v", logsRequested, tt.wantLogs)
}
})
}
}
// writeStderrFrame writes one line as a Docker multiplexed stderr frame, the
// shape ContainerLogs returns for a container without a TTY.
func writeStderrFrame(t *testing.T, w io.Writer, line string) {
t.Helper()
payload := []byte(line + "\n")
frame := [8]byte{2}
binary.BigEndian.PutUint32(frame[4:], uint32(len(payload)))
if _, err := w.Write(append(frame[:], payload...)); err != nil {
t.Error(err)
}
}