Part of #453 (task-observability epic #450). archie-agent logged to os.Stderr only, and its container is created with AutoRemove: true (internal/container/pool.go), so all of it disappeared the moment a task's container exited -- there was no way to see why, say, a gate command failed inside one. Adds agentexec.SystemLogHandler: a slog.Handler that tees to whatever it wraps (kept for an operator attached to a live container) and additionally publishes each record, JSON-encoded as a logging.Entry, to SubjectForSystem(taskID) -- a subject that already existed, documented for exactly this ("log dumps, health... observability"), with zero publishers anywhere in the codebase. cmd/archie-agent/main.go wires it in only when this container was spawned for one specific task (bootTaskID / .git/task.json present); shared queue-group mode serves many unrelated tasks per process, so there's no single task subject to ship to and keeps the stderr-only logger. Building this surfaced a real pre-existing bug: FeedHandler's attr flattening applied the group chain active at Handle-time to *all* accumulated attrs, including ones added via WithAttrs before a later WithGroup call -- nesting them under a group they were never actually inside, contrary to slog's own documented Handler contract. It went unnoticed because nothing in the codebase called WithGroup until this change needed to. Extracting the shared flattening logic into exported FlattenAttrs/PrefixAttrs (so SystemLogHandler doesn't duplicate it) is what surfaced it; fixed in the same change since I was the one making it public API. Docs on the wire format's own trap for the daemon-side consumer (#454, not yet written): the published bytes are a marshaled logging.Entry, meant for json.Unmarshal, not logging.decode() -- decode() parses raw slog JSON *log lines* off disk, a different and incompatible shape. See the doc comment on SystemLogHandler.
175 lines
4.7 KiB
Go
175 lines
4.7 KiB
Go
package logging
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"sync"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// Feed is a logging-owned bounded replay buffer. It is intentionally separate
|
|
// from task activity: daemon diagnostics are not lifecycle events.
|
|
type Feed struct {
|
|
mu sync.RWMutex
|
|
entries []Entry
|
|
limit int
|
|
nextID atomic.Int64
|
|
subs map[chan Entry]struct{}
|
|
}
|
|
|
|
func NewFeed(limit int) *Feed {
|
|
if limit <= 0 {
|
|
limit = 500
|
|
}
|
|
return &Feed{limit: limit, subs: make(map[chan Entry]struct{})}
|
|
}
|
|
|
|
func (f *Feed) Snapshot() []Entry {
|
|
if f == nil {
|
|
return nil
|
|
}
|
|
f.mu.RLock()
|
|
defer f.mu.RUnlock()
|
|
return append([]Entry(nil), f.entries...)
|
|
}
|
|
|
|
func (f *Feed) Subscribe(ctx context.Context) <-chan Entry {
|
|
ch := make(chan Entry, 64)
|
|
if f == nil {
|
|
close(ch)
|
|
return ch
|
|
}
|
|
f.mu.Lock()
|
|
f.subs[ch] = struct{}{}
|
|
f.mu.Unlock()
|
|
go func() {
|
|
<-ctx.Done()
|
|
f.mu.Lock()
|
|
if _, ok := f.subs[ch]; ok {
|
|
delete(f.subs, ch)
|
|
close(ch)
|
|
}
|
|
f.mu.Unlock()
|
|
}()
|
|
return ch
|
|
}
|
|
|
|
func (f *Feed) append(entry Entry) {
|
|
if f == nil {
|
|
return
|
|
}
|
|
entry.ID = f.nextID.Add(1)
|
|
f.mu.Lock()
|
|
if len(f.entries) == f.limit {
|
|
copy(f.entries, f.entries[1:])
|
|
f.entries[len(f.entries)-1] = entry
|
|
} else {
|
|
f.entries = append(f.entries, entry)
|
|
}
|
|
for ch := range f.subs {
|
|
select {
|
|
case ch <- entry:
|
|
default:
|
|
}
|
|
}
|
|
f.mu.Unlock()
|
|
}
|
|
|
|
// FeedHandler writes to the wrapped slog handler and copies the same record to
|
|
// a bounded live feed. The dashboard never parses slog JSON itself.
|
|
type FeedHandler struct {
|
|
next slog.Handler
|
|
feed *Feed
|
|
attrs []slog.Attr
|
|
groups []string
|
|
}
|
|
|
|
func NewFeedHandler(next slog.Handler, feed *Feed) *FeedHandler {
|
|
return &FeedHandler{next: next, feed: feed}
|
|
}
|
|
|
|
func (h *FeedHandler) Enabled(ctx context.Context, level slog.Level) bool {
|
|
return h.next.Enabled(ctx, level)
|
|
}
|
|
|
|
func (h *FeedHandler) Handle(ctx context.Context, record slog.Record) error {
|
|
fields := FlattenAttrs(record, h.attrs, h.groups)
|
|
h.feed.append(Entry{Time: record.Time, Level: record.Level.String(), Message: record.Message, Fields: fields})
|
|
return h.next.Handle(ctx, record)
|
|
}
|
|
|
|
// WithAttrs bakes the current group prefix into each new attr's key
|
|
// immediately, rather than storing it raw and re-deriving the key at
|
|
// Handle-time from whatever the group chain has grown to by then. Deferring
|
|
// it would nest an attr under a group added by a *later* WithGroup call it
|
|
// was never actually inside -- slog's contract is that WithGroup affects
|
|
// only attrs added afterwards.
|
|
func (h *FeedHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
|
clone := *h
|
|
clone.attrs = append(append([]slog.Attr(nil), h.attrs...), PrefixAttrs(attrs, h.groups)...)
|
|
return &clone
|
|
}
|
|
|
|
func (h *FeedHandler) WithGroup(name string) slog.Handler {
|
|
clone := *h
|
|
clone.groups = append(append([]string(nil), h.groups...), name)
|
|
return &clone
|
|
}
|
|
|
|
// PrefixAttrs renames each attr's key with the given dotted group chain, so
|
|
// it carries its grouping with it regardless of what the chain grows to
|
|
// later. Call this from a slog.Handler's WithAttrs, never from Handle: a
|
|
// record's own attrs are always evaluated at the current group nesting, so
|
|
// they're flattened with the live chain instead (see FlattenAttrs).
|
|
// Exported for the same reason as FlattenAttrs: more than one handler needs
|
|
// to get slog's group-nesting contract right (FeedHandler here, and
|
|
// agentexec's system-log handler).
|
|
func PrefixAttrs(attrs []slog.Attr, groups []string) []slog.Attr {
|
|
if len(groups) == 0 {
|
|
return attrs
|
|
}
|
|
out := make([]slog.Attr, len(attrs))
|
|
for i, attr := range attrs {
|
|
key := attr.Key
|
|
for _, group := range groups {
|
|
if group != "" {
|
|
key = group + "." + key
|
|
}
|
|
}
|
|
out[i] = slog.Attr{Key: key, Value: attr.Value}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// FlattenAttrs merges base (handler-bound attrs from WithAttrs, already
|
|
// keyed with whatever group chain was active when each was added -- see
|
|
// prefixAttrs) with a record's own attrs, grouped under the chain currently
|
|
// active at Handle-time, into the single flat map Entry.Fields expects on
|
|
// decode. Exported because more than one slog.Handler needs to produce
|
|
// Entry-shaped output from a slog.Record: FeedHandler here, and agentexec's
|
|
// system-log handler shipping a task's own logs.
|
|
func FlattenAttrs(record slog.Record, base []slog.Attr, groups []string) map[string]any {
|
|
fields := map[string]any{}
|
|
for _, attr := range base {
|
|
addAttr(fields, nil, attr)
|
|
}
|
|
record.Attrs(func(attr slog.Attr) bool { addAttr(fields, groups, attr); return true })
|
|
if len(fields) == 0 {
|
|
return nil
|
|
}
|
|
return fields
|
|
}
|
|
|
|
func addAttr(fields map[string]any, groups []string, attr slog.Attr) {
|
|
if attr.Equal(slog.Attr{}) {
|
|
return
|
|
}
|
|
key := attr.Key
|
|
for _, group := range groups {
|
|
if group != "" {
|
|
key = group + "." + key
|
|
}
|
|
}
|
|
fields[key] = attr.Value.Any()
|
|
}
|