diff --git a/cmd/archie-agent/main.go b/cmd/archie-agent/main.go index 8577fb821..f19ce6f87 100644 --- a/cmd/archie-agent/main.go +++ b/cmd/archie-agent/main.go @@ -106,8 +106,30 @@ func run() int { return 1 } + // A dedicated per-task subscription (below) means this container was + // spawned for one task, which is also when its stderr is otherwise + // unrecoverable: container.Pool creates it with AutoRemove: true. Once + // nc exists, ship this process's own logs to the daemon over + // SubjectForSystem(taskID) so they survive the container's exit. Shared + // queue-group mode serves many unrelated tasks per process, so there is + // no single task subject to ship to -- it keeps the stderr-only logger. + // + // This only covers logging done through the `log` variable from here on. + // bus (arnats.Client) was constructed above with the pre-wrap logger and + // keeps it for its own lifetime, so its own wire-activity Debug lines + // (e.g. "published" on every bus.Respond) stay stderr-only regardless. + // That's a real, known gap, not an oversight: it's the bus's internal + // chatter about NATS itself, not the agent's own reasoning or tool + // output, and reordering construction to close it would mean connecting + // to NATS before knowing whether NATS is even needed for this call. + taskID, hasTaskID := bootTaskID(storage.WorktreeMountDir, log) + if hasTaskID { + log = slog.New(agentexec.NewSystemLogHandler(log.Handler(), nc, taskID)) + log.Info("system log publisher attached", "task", taskID) + } + // Subscribe to task run messages. - taskRunSub, err := subscribeTaskRuns(ctx, nc, log) + taskRunSub, err := subscribeTaskRuns(ctx, nc, log, taskID, hasTaskID) if err != nil { log.Error("taskrun subscribe failed", "err", err) return 1 @@ -124,9 +146,11 @@ func run() int { // subscribeTaskRuns prefers a dedicated per-task subscription over the // shared queue group whenever this container was spawned for a specific -// task (.git/task.json present at the worktree mount). -func subscribeTaskRuns(ctx context.Context, nc *nats.Conn, log *slog.Logger) (*nats.Subscription, error) { - if taskID, ok := bootTaskID(storage.WorktreeMountDir, log); ok { +// task. taskID/hasTaskID come from bootTaskID (.git/task.json present at the +// worktree mount) -- the caller resolves it once, since it also decides +// whether to attach a per-task log publisher to log before this is called. +func subscribeTaskRuns(ctx context.Context, nc *nats.Conn, log *slog.Logger, taskID int64, hasTaskID bool) (*nats.Subscription, error) { + if hasTaskID { log.Info("taskrun: dedicated per-task subscription", "task", taskID) return nc.Subscribe(taskrun.SubjectForTask(taskID), func(msg *nats.Msg) { handleTaskRun(ctx, msg, nc, log) diff --git a/internal/agentexec/systemlog.go b/internal/agentexec/systemlog.go new file mode 100644 index 000000000..6558eadc8 --- /dev/null +++ b/internal/agentexec/systemlog.go @@ -0,0 +1,120 @@ +package agentexec + +import ( + "context" + "encoding/json" + "log/slog" + "sync" + "time" + + "github.com/samcharles93/archie-core/internal/logging" +) + +// LogPublisher is the minimal capability SystemLogHandler needs: fire a +// message at a subject without waiting for a reply. Declared here rather +// than taking *nats.go's Conn or eventbus.Bus so the handler can be tested +// with a fake and never gains capabilities (subscribe, request/reply) it has +// no use for. *nats.Conn already satisfies this with no adapter needed. +type LogPublisher interface { + Publish(subject string, data []byte) error +} + +// SystemLogHandler tees slog records to a wrapped handler -- normally the +// agent's own stderr, kept for an operator attached to a live container -- +// and additionally publishes each one, JSON-encoded as a logging.Entry, to +// SubjectForSystem(taskID): the return channel documented in subjects.go for +// "log dumps, health... observability" that previously had no publisher. +// +// The wire payload is a marshaled logging.Entry -- {"time","level","msg", +// "fields":{...}} -- meant to be round-tripped with json.Unmarshal straight +// back into a logging.Entry. This is deliberately NOT the same shape +// logging.decode() parses: decode() reads raw slog JSON *log lines* off +// disk, where every non-{time,level,msg} key sits at the top level and +// decode() moves it into Fields itself. Handing decode() this payload +// instead of json.Unmarshal would nest everything one level too deep, under +// a literal "fields" key, and silently lose every real field name. A future +// consumer of this subject (see GitHub issue #454) must unmarshal directly +// into logging.Entry, not call decode() on these bytes. +// +// Publishing is fire-and-forget. A core NATS Publish call is already +// non-blocking -- it queues to the connection's outbound buffer and returns, +// and the client itself drops writes rather than blocking once its +// reconnect buffer is full -- so this handler does not duplicate that +// behaviour with a second, home-grown bounded queue. A publish or marshal +// failure is reported once via the wrapped handler and then swallowed: log +// shipping must never block or fail the run it is reporting on, and a dead +// NATS connection is already visible through every stage subsequently +// failing to report its own result. +type SystemLogHandler struct { + next slog.Handler + pub LogPublisher + subject string + + attrs []slog.Attr + groups []string + + // warnOnce is a pointer, shared across every clone WithAttrs/WithGroup + // produce from one root handler: sync.Once is not safe to copy by value + // (go vet: copylocks), and the "log the publish failure once" intent is + // per underlying connection, not per derived logger. + warnOnce *sync.Once +} + +// NewSystemLogHandler wraps next for taskID's system log subject. +func NewSystemLogHandler(next slog.Handler, pub LogPublisher, taskID int64) *SystemLogHandler { + return &SystemLogHandler{next: next, pub: pub, subject: SubjectForSystem(taskID), warnOnce: &sync.Once{}} +} + +func (h *SystemLogHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.next.Enabled(ctx, level) +} + +func (h *SystemLogHandler) Handle(ctx context.Context, record slog.Record) error { + entry := logging.Entry{ + Time: record.Time, + Level: record.Level.String(), + Message: record.Message, + Fields: logging.FlattenAttrs(record, h.attrs, h.groups), + } + data, err := json.Marshal(entry) + if err == nil { + err = h.pub.Publish(h.subject, data) + } + if err != nil { + h.warnAtMostOnce(ctx, record.Time, err) + } + return h.next.Handle(ctx, record) +} + +// warnAtMostOnce reports the first system-log delivery failure (marshal or +// publish, either way this record's own history is now missing on the +// daemon side) via the wrapped handler, then goes quiet -- a dead +// connection would otherwise turn every subsequent line into a second +// failure notice on stderr, drowning out the output an operator is trying +// to read. +func (h *SystemLogHandler) warnAtMostOnce(ctx context.Context, at time.Time, cause error) { + h.warnOnce.Do(func() { + warning := slog.NewRecord(at, slog.LevelWarn, + "system log delivery failed; further failures on this connection are not logged individually", 0) + warning.AddAttrs(slog.String("err", cause.Error())) + _ = h.next.Handle(ctx, warning) + }) +} + +// WithAttrs bakes the current group prefix into each new attr's key +// immediately -- see logging.PrefixAttrs' sibling logic in feed.go for why +// deferring it to Handle-time is wrong: it would nest an attr under a group +// added by a *later* WithGroup call it was never actually inside. +func (h *SystemLogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + clone := *h + clone.next = h.next.WithAttrs(attrs) + clone.attrs = append(append([]slog.Attr(nil), h.attrs...), logging.PrefixAttrs(attrs, h.groups)...) + return &clone +} + +func (h *SystemLogHandler) WithGroup(name string) slog.Handler { + clone := *h + clone.next = h.next.WithGroup(name) + clone.groups = append(append([]string(nil), h.groups...), name) + return &clone +} diff --git a/internal/agentexec/systemlog_test.go b/internal/agentexec/systemlog_test.go new file mode 100644 index 000000000..2622b48ee --- /dev/null +++ b/internal/agentexec/systemlog_test.go @@ -0,0 +1,226 @@ +package agentexec + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "sync" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/logging" +) + +// fakePublisher is a LogPublisher that records every call and can be told to +// fail, so tests can assert both the happy path and the "must never block or +// fail the run" requirement. +type fakePublisher struct { + mu sync.Mutex + subjects []string + payloads [][]byte + failWith error +} + +func (f *fakePublisher) Publish(subject string, data []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + f.subjects = append(f.subjects, subject) + f.payloads = append(f.payloads, append([]byte(nil), data...)) + return f.failWith +} + +func (f *fakePublisher) calls() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.payloads) +} + +// capturingHandler is a minimal slog.Handler that records every record it +// receives, standing in for cmd/archie-agent's real stderr handler. +type capturingHandler struct { + mu sync.Mutex + records []slog.Record +} + +func (h *capturingHandler) Enabled(context.Context, slog.Level) bool { return true } + +func (h *capturingHandler) Handle(_ context.Context, r slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, r) + return nil +} + +func (h *capturingHandler) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *capturingHandler) WithGroup(string) slog.Handler { return h } + +func (h *capturingHandler) messages() []string { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]string, len(h.records)) + for i, r := range h.records { + out[i] = r.Message + } + return out +} + +func TestSystemLogHandlerPublishesToTheTasksSystemSubject(t *testing.T) { + pub := &fakePublisher{} + next := &capturingHandler{} + handler := NewSystemLogHandler(next, pub, 42) + logger := slog.New(handler) + + logger.Warn("gate failed", "component", "gate") + + if got, want := pub.calls(), 1; got != want { + t.Fatalf("publish calls = %d, want %d", got, want) + } + if got, want := pub.subjects[0], SubjectForSystem(42); got != want { + t.Errorf("subject = %q, want %q", got, want) + } + + var entry logging.Entry + if err := json.Unmarshal(pub.payloads[0], &entry); err != nil { + t.Fatalf("payload is not a logging.Entry: %v", err) + } + if entry.Message != "gate failed" { + t.Errorf("Message = %q, want %q", entry.Message, "gate failed") + } + if entry.Level != "WARN" { + t.Errorf("Level = %q, want WARN", entry.Level) + } + if entry.Fields["component"] != "gate" { + t.Errorf("Fields[component] = %v, want gate", entry.Fields["component"]) + } +} + +func TestSystemLogHandlerAlwaysTeesToNextRegardlessOfPublishOutcome(t *testing.T) { + tests := []struct { + name string + failWith error + }{ + {name: "publish succeeds"}, + {name: "publish fails", failWith: errors.New("nats: no responders")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pub := &fakePublisher{failWith: tt.failWith} + next := &capturingHandler{} + logger := slog.New(NewSystemLogHandler(next, pub, 1)) + + logger.Info("still visible on stderr") + + // A failed publish also emits its own one-time warning onto + // next (see TestSystemLogHandlerLogsPublishFailureOnceNotPerRecord), + // so check membership, not position. + found := false + for _, msg := range next.messages() { + found = found || msg == "still visible on stderr" + } + if !found { + t.Errorf("next handler messages = %v, want %q present regardless of publish outcome", + next.messages(), "still visible on stderr") + } + }) + } +} + +// TestSystemLogHandlerNeverFailsTheRunOnPublishError guards the explicit +// design requirement: log shipping must never block or fail the run it is +// reporting on. A slog.Logger call that returned an error here would +// propagate nowhere useful (slog callers never check it), but Handle itself +// must not return a publish error either, or a handler chain that does check +// (as ours might grow to) would start treating a dead NATS connection as a +// logging failure worth surfacing to the caller. +func TestSystemLogHandlerNeverFailsTheRunOnPublishError(t *testing.T) { + pub := &fakePublisher{failWith: errors.New("nats: no responders")} + handler := NewSystemLogHandler(&capturingHandler{}, pub, 1) + + rec := slog.NewRecord(time.Now(), slog.LevelInfo, "msg", 0) + if err := handler.Handle(context.Background(), rec); err != nil { + t.Errorf("Handle() = %v, want nil even when the publisher errors", err) + } +} + +// TestSystemLogHandlerLogsPublishFailureOnceNotPerRecord guards against a +// dead NATS connection turning every subsequent log line into a second +// failure notice on stderr, which would drown out the very output the +// operator is trying to read. +func TestSystemLogHandlerLogsPublishFailureOnceNotPerRecord(t *testing.T) { + pub := &fakePublisher{failWith: errors.New("nats: no responders")} + next := &capturingHandler{} + logger := slog.New(NewSystemLogHandler(next, pub, 1)) + + for range 5 { + logger.Info("line") + } + + failureNotices := 0 + for _, msg := range next.messages() { + if msg != "line" { + failureNotices++ + } + } + if failureNotices != 1 { + t.Errorf("failure notices on next = %d, want exactly 1 across 5 failed publishes", failureNotices) + } +} + +// TestSystemLogHandlerMarshalFailureWarnsOnceAndNeverPublishes guards the +// other delivery failure mode alongside a rejected Publish: an attr value +// json.Marshal can't encode (e.g. a channel) must not silently vanish with +// zero signal, and must not reach the publisher with malformed data either. +func TestSystemLogHandlerMarshalFailureWarnsOnceAndNeverPublishes(t *testing.T) { + pub := &fakePublisher{} + next := &capturingHandler{} + logger := slog.New(NewSystemLogHandler(next, pub, 1)) + + logger.Info("unmarshalable", "bad", make(chan int)) + + if pub.calls() != 0 { + t.Errorf("publish calls = %d, want 0: malformed data must never reach the publisher", pub.calls()) + } + + failureNotices := 0 + for _, msg := range next.messages() { + if msg != "unmarshalable" { + failureNotices++ + } + } + if failureNotices != 1 { + t.Errorf("failure notices on next = %d, want exactly 1", failureNotices) + } +} + +func TestSystemLogHandlerWithAttrsAndGroupsReachThePublishedEntry(t *testing.T) { + pub := &fakePublisher{} + next := &capturingHandler{} + handler := NewSystemLogHandler(next, pub, 7) + + logger := slog.New(handler).With("task", "7").WithGroup("gate") + logger.Warn("failed", "cmd", "go build ./...") + + if pub.calls() != 1 { + t.Fatalf("publish calls = %d, want 1", pub.calls()) + } + var entry logging.Entry + if err := json.Unmarshal(pub.payloads[0], &entry); err != nil { + t.Fatalf("payload is not a logging.Entry: %v", err) + } + if entry.Fields["task"] != "7" { + t.Errorf("Fields[task] = %v, want 7 (WithAttrs before WithGroup keeps its own key ungrouped)", entry.Fields["task"]) + } + if entry.Fields["gate.cmd"] != "go build ./..." { + t.Errorf("Fields[gate.cmd] = %v, want %q", entry.Fields["gate.cmd"], "go build ./...") + } +} + +func TestSystemLogHandlerEnabledDelegatesToNext(t *testing.T) { + next := &capturingHandler{} + handler := NewSystemLogHandler(next, &fakePublisher{}, 1) + if !handler.Enabled(context.Background(), slog.LevelInfo) { + t.Error("Enabled() = false, want true (next always returns true)") + } +} diff --git a/internal/logging/attrs_test.go b/internal/logging/attrs_test.go new file mode 100644 index 000000000..4c00db325 --- /dev/null +++ b/internal/logging/attrs_test.go @@ -0,0 +1,83 @@ +package logging + +import ( + "log/slog" + "testing" + "time" +) + +// TestFlattenAttrs pins the shape FeedHandler and any other slog.Handler +// producing Entry.Fields must agree on: handler-bound attrs, group prefixes, +// and the record's own attrs all merge into one flat map, dotted by group. +func TestFlattenAttrs(t *testing.T) { + tests := []struct { + name string + base []slog.Attr + groups []string + record func() slog.Record + want map[string]any + }{ + { + name: "record attrs only", + record: func() slog.Record { return newTestRecord("component", "gate") }, + want: map[string]any{"component": "gate"}, + }, + { + name: "base attrs merge with record attrs", + base: []slog.Attr{slog.String("task", "42")}, + record: func() slog.Record { return newTestRecord("component", "gate") }, + want: map[string]any{"task": "42", "component": "gate"}, + }, + { + name: "groups prefix every key", + groups: []string{"agent"}, + record: func() slog.Record { return newTestRecord("component", "gate") }, + want: map[string]any{"agent.component": "gate"}, + }, + { + name: "no attrs at all yields nil, not an empty map", + record: func() slog.Record { return newTestRecord() }, + want: nil, + }, + { + // base is what WithAttrs accumulates; by the time it reaches + // FlattenAttrs it must already carry whatever group prefix was + // active when it was added (see prefixAttrs) -- groups here is + // the chain active *now*, for the record's own attrs, and must + // not be re-applied to base. Callers that instead pass raw, + // un-prefixed attrs as base alongside a non-empty groups chain + // are the historical bug this case pins: it would wrongly nest + // "task" under "agent" too. + name: "groups only apply to the record's own attrs, not to base", + base: []slog.Attr{slog.String("task", "42")}, // pre-grouped by the caller; here, not grouped at all + groups: []string{"agent"}, + record: func() slog.Record { return newTestRecord("component", "gate") }, + want: map[string]any{"task": "42", "agent.component": "gate"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FlattenAttrs(tt.record(), tt.base, tt.groups) + if len(got) != len(tt.want) { + t.Fatalf("FlattenAttrs() = %#v, want %#v", got, tt.want) + } + for k, v := range tt.want { + if got[k] != v { + t.Errorf("FlattenAttrs()[%q] = %v, want %v", k, got[k], v) + } + } + if tt.want == nil && got != nil { + t.Errorf("FlattenAttrs() = %#v, want nil", got) + } + }) + } +} + +func newTestRecord(kvs ...string) slog.Record { + r := slog.NewRecord(time.Now(), slog.LevelInfo, "msg", 0) + for i := 0; i+1 < len(kvs); i += 2 { + r.AddAttrs(slog.String(kvs[i], kvs[i+1])) + } + return r +} diff --git a/internal/logging/feed.go b/internal/logging/feed.go index 0e8e595a5..7f2b655fb 100644 --- a/internal/logging/feed.go +++ b/internal/logging/feed.go @@ -93,21 +93,20 @@ func (h *FeedHandler) Enabled(ctx context.Context, level slog.Level) bool { } func (h *FeedHandler) Handle(ctx context.Context, record slog.Record) error { - fields := map[string]any{} - for _, attr := range h.attrs { - addAttr(fields, h.groups, attr) - } - record.Attrs(func(attr slog.Attr) bool { addAttr(fields, h.groups, attr); return true }) - if len(fields) == 0 { - fields = nil - } + fields := FlattenAttrs(record, h.attrs, h.groups) h.feed.append(Entry{Time: record.Time, Level: record.Level.String(), Message: record.Message, Fields: fields}) return h.next.Handle(ctx, record) } +// WithAttrs bakes the current group prefix into each new attr's key +// immediately, rather than storing it raw and re-deriving the key at +// Handle-time from whatever the group chain has grown to by then. Deferring +// it would nest an attr under a group added by a *later* WithGroup call it +// was never actually inside -- slog's contract is that WithGroup affects +// only attrs added afterwards. func (h *FeedHandler) WithAttrs(attrs []slog.Attr) slog.Handler { clone := *h - clone.attrs = append(append([]slog.Attr(nil), h.attrs...), attrs...) + clone.attrs = append(append([]slog.Attr(nil), h.attrs...), PrefixAttrs(attrs, h.groups)...) return &clone } @@ -117,6 +116,50 @@ func (h *FeedHandler) WithGroup(name string) slog.Handler { return &clone } +// PrefixAttrs renames each attr's key with the given dotted group chain, so +// it carries its grouping with it regardless of what the chain grows to +// later. Call this from a slog.Handler's WithAttrs, never from Handle: a +// record's own attrs are always evaluated at the current group nesting, so +// they're flattened with the live chain instead (see FlattenAttrs). +// Exported for the same reason as FlattenAttrs: more than one handler needs +// to get slog's group-nesting contract right (FeedHandler here, and +// agentexec's system-log handler). +func PrefixAttrs(attrs []slog.Attr, groups []string) []slog.Attr { + if len(groups) == 0 { + return attrs + } + out := make([]slog.Attr, len(attrs)) + for i, attr := range attrs { + key := attr.Key + for _, group := range groups { + if group != "" { + key = group + "." + key + } + } + out[i] = slog.Attr{Key: key, Value: attr.Value} + } + return out +} + +// FlattenAttrs merges base (handler-bound attrs from WithAttrs, already +// keyed with whatever group chain was active when each was added -- see +// prefixAttrs) with a record's own attrs, grouped under the chain currently +// active at Handle-time, into the single flat map Entry.Fields expects on +// decode. Exported because more than one slog.Handler needs to produce +// Entry-shaped output from a slog.Record: FeedHandler here, and agentexec's +// system-log handler shipping a task's own logs. +func FlattenAttrs(record slog.Record, base []slog.Attr, groups []string) map[string]any { + fields := map[string]any{} + for _, attr := range base { + addAttr(fields, nil, attr) + } + record.Attrs(func(attr slog.Attr) bool { addAttr(fields, groups, attr); return true }) + if len(fields) == 0 { + return nil + } + return fields +} + func addAttr(fields map[string]any, groups []string, attr slog.Attr) { if attr.Equal(slog.Attr{}) { return