fix(container): report a crashed container's panic line and log its stderr
This commit is contained in:
1 parent
d7d1bd4556
commit
4dca007003
3 files changed
+185
-14
No files matched your search
@@ -269,13 +269,13 @@ func parseListenAddr(addr, defaultHost string, defaultPort int) (string, int) {
|
||||
}
|
||||
|
||||
// subscribeSystemLogs subscribes to every task's system log subject at once.
|
||||
// A sandboxed container's own stderr disappears at AutoRemove; archie-agent
|
||||
// ships it here instead (agentexec.SubjectForSystem), and this is where it
|
||||
// lands: one wildcard subscription demuxed by taskID and routed through
|
||||
// taskLogs, which is opened for a task's duration in Daemon.process, so a
|
||||
// message for a task not currently open there -- late, duplicate, or from
|
||||
// an attempt this instance never dispatched -- is expected and silently
|
||||
// dropped rather than treated as an error.
|
||||
// archie-agent's own log lines ship here (agentexec.SubjectForSystem) rather
|
||||
// than staying on its stderr, and this is where they land: one wildcard
|
||||
// subscription demuxed by taskID and routed through taskLogs, which is opened
|
||||
// for a task's duration in Daemon.process, so a message for a task not
|
||||
// currently open there -- late, duplicate, or from an attempt this instance
|
||||
// never dispatched -- is expected and silently dropped rather than treated as
|
||||
// an error.
|
||||
func subscribeSystemLogs(nc *natsio.Conn, taskLogs *logging.TaskRegistry, log *slog.Logger) (unsubscribe func(), err error) {
|
||||
sub, err := nc.Subscribe(agentexec.SubjectSystemWildcard, func(msg *natsio.Msg) {
|
||||
taskID, ok := agentexec.TaskIDFromSystemSubject(msg.Subject)
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
@@ -15,9 +16,12 @@ import (
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/registry"
|
||||
"github.com/moby/moby/client"
|
||||
@@ -39,8 +43,9 @@ type Container struct {
|
||||
// not start returns nil, which never closes.
|
||||
func (c *Container) Exited() <-chan struct{} { return c.exited }
|
||||
|
||||
// ExitDetail says how the container ended: its exit code. It is empty until
|
||||
// Exited closes, and for a container the pool did not start.
|
||||
// ExitDetail says how the container ended: its exit code, and the runtime's
|
||||
// crash line when it panicked. It is empty until Exited closes, and for a
|
||||
// container the pool did not start.
|
||||
func (c *Container) ExitDetail() string {
|
||||
if c.detail == nil {
|
||||
return ""
|
||||
@@ -260,7 +265,10 @@ func (p *Pool) Acquire(ctx context.Context, image string, mounts []storage.Mount
|
||||
|
||||
name := fmt.Sprintf("%s-agent-%d", cfg.Instance, time.Now().UnixNano())
|
||||
|
||||
hostConfig := &container.HostConfig{AutoRemove: true}
|
||||
// Release owns removal rather than Docker's auto-remove: a crash writes its
|
||||
// panic trace to stderr, and the container's log is unreadable the moment
|
||||
// Docker removes it.
|
||||
hostConfig := &container.HostConfig{}
|
||||
for i, m := range storage.ConvertMounts(mounts) {
|
||||
if string(m.Type) == storage.MountTypeBind {
|
||||
label := "Z"
|
||||
@@ -374,7 +382,7 @@ func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan
|
||||
go func() {
|
||||
select {
|
||||
case result := <-wait.Result:
|
||||
*detail = exitDetail(result.StatusCode)
|
||||
*detail = p.exitDetail(ctx, id, result.StatusCode)
|
||||
close(exited)
|
||||
case <-wait.Error:
|
||||
}
|
||||
@@ -382,9 +390,80 @@ func (p *Pool) watchExit(ctx context.Context, id string, detail *string) <-chan
|
||||
return exited
|
||||
}
|
||||
|
||||
// exitDetail describes a finished container for the operator. Its output is
|
||||
// left out: the reason reaches the dashboard, logs and notifications, and an
|
||||
// agent's last lines can carry a credential.
|
||||
// exitDetail describes a finished container for the operator. The detail
|
||||
// becomes the task's park reason, which reaches the dashboard and chat, and an
|
||||
// agent's output can carry a credential, so it takes only the crash line. The
|
||||
// full stderr tail, the only account of a Go panic, goes to the daemon log.
|
||||
func (p *Pool) exitDetail(ctx context.Context, id string, code int64) string {
|
||||
detail := exitDetail(code)
|
||||
if code == 0 {
|
||||
return detail
|
||||
}
|
||||
tail := p.stderrTail(ctx, id)
|
||||
if tail == "" {
|
||||
return detail
|
||||
}
|
||||
p.log.ErrorContext(ctx, "agent container crashed", "id", id[:12], "code", code, "stderr", tail)
|
||||
if line := crashLine(tail); line != "" {
|
||||
detail += ": " + line
|
||||
}
|
||||
return detail
|
||||
}
|
||||
|
||||
// crashLineBytes bounds the crash line carried into the park reason.
|
||||
const crashLineBytes = 200
|
||||
|
||||
// crashLine picks the runtime's own summary of a crash from a stderr tail.
|
||||
func crashLine(tail string) string {
|
||||
for line := range strings.Lines(tail) {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "panic: ") || strings.HasPrefix(line, "fatal error: ") {
|
||||
if len(line) > crashLineBytes {
|
||||
line = line[:crashLineBytes] + "…"
|
||||
}
|
||||
return line
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
const (
|
||||
// crashTailLines bounds the stderr tail read after a crash: enough for a
|
||||
// panic's opening line and the frames around the fault.
|
||||
crashTailLines = 100
|
||||
// crashTailBytes bounds the same tail by size, for a container that writes
|
||||
// one enormous line or a multi-megabyte stack dump.
|
||||
crashTailBytes = 8 << 10
|
||||
)
|
||||
|
||||
// stderrTail returns what the container last wrote to stderr. It reads the log
|
||||
// of a stopped container, which exists until Release removes it. Best effort: a
|
||||
// container with no output, or a Docker error, yields "".
|
||||
func (p *Pool) stderrTail(ctx context.Context, id string) string {
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
defer cancel()
|
||||
reader, err := p.cli.ContainerLogs(ctx, id, client.ContainerLogsOptions{
|
||||
ShowStderr: true,
|
||||
Tail: strconv.Itoa(crashTailLines),
|
||||
})
|
||||
if err != nil {
|
||||
p.log.WarnContext(ctx, "container stderr unavailable after exit", "id", id[:12], "err", err)
|
||||
return ""
|
||||
}
|
||||
defer func() {
|
||||
if err := reader.Close(); err != nil {
|
||||
p.log.WarnContext(ctx, "container log reader close failed", "id", id[:12], "err", err)
|
||||
}
|
||||
}()
|
||||
var stderr bytes.Buffer
|
||||
// stdout is not part of the detail. Hitting the byte cap cuts the stream
|
||||
// short, so the partial text is kept rather than the read reported as a
|
||||
// failure.
|
||||
_, _ = stdcopy.StdCopy(io.Discard, &stderr, io.LimitReader(reader, crashTailBytes))
|
||||
return strings.TrimSpace(stderr.String())
|
||||
}
|
||||
|
||||
// exitDetail is the operator-facing one-line cause of a container's exit.
|
||||
func exitDetail(code int64) string {
|
||||
if code == 137 {
|
||||
return "exit code 137 (killed; often out of memory)"
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -9,6 +12,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
@@ -74,3 +78,91 @@ func TestPoolReapsOnlyOwnInstance(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestExitDetailCarriesCrashStderr runs the exit watch against a fake Engine: a
|
||||
// container that fails writes its panic trace to stderr, and the panic line has
|
||||
// to reach the exit detail before Release removes the container, without the
|
||||
// rest of its output.
|
||||
func TestExitDetailCarriesCrashStderr(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
code int64
|
||||
stderr []string
|
||||
want []string
|
||||
reject []string
|
||||
wantLogs bool
|
||||
}{
|
||||
{
|
||||
name: "crash keeps only the panic line",
|
||||
code: 2,
|
||||
stderr: []string{"token=s3cret", "panic: runtime error: boom", "goroutine 1 [running]:"},
|
||||
want: []string{"exit code 2", "panic: runtime error: boom"},
|
||||
reject: []string{"s3cret", "goroutine"},
|
||||
wantLogs: true,
|
||||
},
|
||||
{
|
||||
name: "clean exit asks for no log",
|
||||
code: 0,
|
||||
stderr: []string{"finished"},
|
||||
want: []string{"exit code 0"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var logsRequested bool
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/wait"):
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"StatusCode":%d}`, tt.code)
|
||||
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/logs"):
|
||||
logsRequested = true
|
||||
for _, line := range tt.stderr {
|
||||
writeStderrFrame(t, w, line)
|
||||
}
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
cli, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.54"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cli.Close()
|
||||
p := &Pool{cli: cli, log: slog.New(slog.DiscardHandler)}
|
||||
detail := new(string)
|
||||
exited := p.watchExit(t.Context(), "crash00000000", detail)
|
||||
select {
|
||||
case <-exited:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("exit watch did not report the container's exit")
|
||||
}
|
||||
for _, want := range tt.want {
|
||||
if !strings.Contains(*detail, want) {
|
||||
t.Fatalf("exit detail %q does not contain %q", *detail, want)
|
||||
}
|
||||
}
|
||||
for _, reject := range tt.reject {
|
||||
if strings.Contains(*detail, reject) {
|
||||
t.Fatalf("exit detail %q carries %q, which belongs only in the daemon log", *detail, reject)
|
||||
}
|
||||
}
|
||||
if logsRequested != tt.wantLogs {
|
||||
t.Fatalf("log requested = %v, want %v", logsRequested, tt.wantLogs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeStderrFrame writes one line as a Docker multiplexed stderr frame, the
|
||||
// shape ContainerLogs returns for a container without a TTY.
|
||||
func writeStderrFrame(t *testing.T, w io.Writer, line string) {
|
||||
t.Helper()
|
||||
payload := []byte(line + "\n")
|
||||
frame := [8]byte{2}
|
||||
binary.BigEndian.PutUint32(frame[4:], uint32(len(payload)))
|
||||
if _, err := w.Write(append(frame[:], payload...)); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user