From 951ddbc9f4a82eeef1181d1ff549822a48a1e1f8 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sun, 9 Aug 2026 18:42:54 +1000 Subject: [PATCH] feat(daemon): consume agent system logs into per-task sinks and the dashboard feed Closes the loop #451-#453 opened. archie-agent (#453) now publishes its own logs to agentexec.SubjectForSystem(taskID) over NATS, and a task log sink (#452) exists to persist them, but nothing on the daemon side subscribed, demuxed, or wrote anything -- the messages had nowhere to land. - agentexec: SubjectSystemWildcard + TaskIDFromSystemSubject, the inverse of SubjectForSystem, for a daemon subscribing to every task's system subject with one wildcard subscription. - logging: TaskRegistry manages one open TaskSink per running task attempt, mirroring every write onto the live dashboard Feed the same way a locally-written log would appear. Nil-receiver-safe, matching every other optional composition-wired Daemon field. - daemon: Daemon.process opens a task's log sink for its own duration (covering both the sandboxed-container and in-process dispatch paths uniformly) and closes it on every return path. - cmd/archied: constructs the registry, subscribes core NATS to SubjectSystemWildcard, and wires TaskLogs into both Daemon and webui.Server. - webui: archiving a task now removes its log directory too, so archived tasks' logs don't accumulate forever. Documented as a known, accepted race: a task can reach terminal status (satisfying ArchiveTask's precondition) before process() itself returns and closes the sink, so a very late system-log write can race an archive's Remove. Not corrupting -- the write lands on a soon-to-be-unlinked inode and is discarded -- and not worth fixing here: it would mean tying Close to the store transition instead of to process()'s return, a larger change than this issue's scope. Adversarial review (fresh subagent) found and fixed two real issues in the first draft: a doc comment accidentally spliced between two functions during an extraction meant to fix a lint regression, and a genuine test gap on openTaskLog (added, using TaskRegistry.Write's own true/false contract as the observable -- no Store/Forge/Trees fakes needed, since openTaskLog only touches TaskLogs and Log). --- cmd/archied/main.go | 48 +++++- internal/agentexec/subjects.go | 30 +++- internal/agentexec/subjects_test.go | 49 ++++++ internal/daemon/daemon.go | 27 ++++ internal/daemon/daemon_test.go | 25 +++ internal/logging/registry.go | 122 +++++++++++++++ internal/logging/registry_test.go | 191 +++++++++++++++++++++++ internal/logging/tasksink.go | 16 +- internal/webui/api_tasks.go | 17 ++ internal/webui/api_tasks_actions_test.go | 40 +++++ internal/webui/server.go | 4 + 11 files changed, 561 insertions(+), 8 deletions(-) create mode 100644 internal/agentexec/subjects_test.go create mode 100644 internal/logging/registry.go create mode 100644 internal/logging/registry_test.go diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 7dff8a4fe..e77671441 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -7,6 +7,7 @@ package main import ( "context" "crypto/sha256" + "encoding/json" "flag" "fmt" "log/slog" @@ -308,6 +309,11 @@ func run() int { // daemon continues on stderr -- losing the durable copy must not take the // daemon down with it. logFeed := logging.NewFeed(1000) + // Task logs live alongside the store rather than under cfg.Log.File's + // directory: cfg.Log.File is optional (file logging can be off), while + // DBPath is required for the daemon to run at all, so it is the more + // reliable anchor for "where archie keeps its state" on this host. + taskLogs := logging.NewTaskRegistry(filepath.Join(filepath.Dir(cfg.DBPath), "logs", "tasks"), logFeed, logging.TaskSinkOptions{}) fileLog, logCloser, logErr := logging.New(logging.Options{ File: cfg.Log.File, MaxSizeMB: cfg.Log.MaxSizeMB, @@ -390,7 +396,7 @@ func run() int { Path: origin.Path, Role: string(origin.Role), Layer: string(origin.Layer), Feature: string(origin.Feature), }) } - web := &webui.Server{Store: st, Log: log, LogFeed: logFeed, Cfg: &cfg, ConfigProvenance: configProvenance, Channels: channelManager, Events: bus} + web := &webui.Server{Store: st, Log: log, LogFeed: logFeed, TaskLogs: taskLogs, Cfg: &cfg, ConfigProvenance: configProvenance, Channels: channelManager, Events: bus} web.ReloadChannel = func(ctx context.Context, id string) error { if id != "telegram" || restartTelegram == nil { return fmt.Errorf("channel reload unavailable") @@ -717,6 +723,13 @@ func run() int { return 1 } defer unsubscribe() + + unsubscribeSystemLogs, err := subscribeSystemLogs(coreConn, taskLogs, log) + if err != nil { + log.Error("system log subscribe failed", "err", err) + return 1 + } + defer unsubscribeSystemLogs() } // ── Memory manager ──────────────────────────────────────────────── @@ -917,6 +930,7 @@ func run() int { ToolRegistry: toolReg, Curators: curatorRegistry, Identities: identityRunners, + TaskLogs: taskLogs, } web.TaskStopper = d @@ -1128,6 +1142,38 @@ func executionProviders(cfg config.Config) map[string]agentexec.Provider { return agentexec.ProvidersFromConfig(cfg.Providers) } +// subscribeSystemLogs subscribes to every task's system log subject at once. +// A sandboxed container's own stderr disappears at AutoRemove; archie-agent +// ships it here instead (agentexec.SubjectForSystem), and this is where it +// lands: one wildcard subscription demuxed by taskID and routed through +// taskLogs, which is opened for a task's duration in Daemon.process, so a +// message for a task not currently open there -- late, duplicate, or from +// an attempt this instance never dispatched -- is expected and silently +// dropped rather than treated as an error. +func subscribeSystemLogs(nc *natsio.Conn, taskLogs *logging.TaskRegistry, log *slog.Logger) (unsubscribe func(), err error) { + sub, err := nc.Subscribe(agentexec.SubjectSystemWildcard, func(msg *natsio.Msg) { + taskID, ok := agentexec.TaskIDFromSystemSubject(msg.Subject) + if !ok { + log.Warn("system log message on unparseable subject", "subject", msg.Subject) + return + } + var entry logging.Entry + if err := json.Unmarshal(msg.Data, &entry); err != nil { + log.Warn("system log message undecodable", "task", taskID, "err", err) + return + } + taskLogs.Write(taskID, entry) + }) + if err != nil { + return nil, err + } + return func() { + if err := sub.Unsubscribe(); err != nil { + log.Warn("system log unsubscribe failed", "err", err) + } + }, nil +} + // registerTaskRPCServers subscribes the storerpc/forgerpc/worktreerpc // handlers on nc so an archie-agent container (which holds no DB // connection, forge token, or push credential) can proxy those operations diff --git a/internal/agentexec/subjects.go b/internal/agentexec/subjects.go index d31308676..352fa6314 100644 --- a/internal/agentexec/subjects.go +++ b/internal/agentexec/subjects.go @@ -1,6 +1,9 @@ package agentexec -import "strconv" +import ( + "strconv" + "strings" +) // Agent execution subjects. // @@ -16,7 +19,13 @@ const ( // SubjectAgentWildcard matches every agent subject. SubjectAgentWildcard = "archie.agent.>" - subjectAgentPrefix = "archie.agent." + // SubjectSystemWildcard matches every task's system subject, for a + // daemon subscribing to all of them at once with a single core NATS + // subscription rather than one per running task. + SubjectSystemWildcard = subjectAgentPrefix + "*.system" + + subjectAgentPrefix = "archie.agent." + subjectSystemSuffix = ".system" ) // SubjectForRequest returns the subject carrying stage execution requests for @@ -43,3 +52,20 @@ func SubjectForSystem(taskID int64) string { func subjectForTask(taskID int64, kind string) string { return subjectAgentPrefix + strconv.FormatInt(taskID, 10) + "." + kind } + +// TaskIDFromSystemSubject extracts the task ID from a subject produced by +// SubjectForSystem, for a daemon demuxing a SubjectSystemWildcard +// subscription. Rejects (0, false) for anything that isn't exactly that +// shape, rather than parsing a partial or wrong-kind subject into a +// misleading task ID. +func TaskIDFromSystemSubject(subject string) (int64, bool) { + if !strings.HasPrefix(subject, subjectAgentPrefix) || !strings.HasSuffix(subject, subjectSystemSuffix) { + return 0, false + } + middle := strings.TrimSuffix(strings.TrimPrefix(subject, subjectAgentPrefix), subjectSystemSuffix) + id, err := strconv.ParseInt(middle, 10, 64) + if err != nil || id <= 0 { + return 0, false + } + return id, true +} diff --git a/internal/agentexec/subjects_test.go b/internal/agentexec/subjects_test.go new file mode 100644 index 000000000..b0cb257a9 --- /dev/null +++ b/internal/agentexec/subjects_test.go @@ -0,0 +1,49 @@ +package agentexec + +import "testing" + +// TestTaskIDFromSystemSubject pins the round trip with SubjectForSystem, and +// that a subject which merely resembles one (wrong kind suffix, malformed +// ID) is rejected rather than parsed into a wrong task ID -- a daemon +// demuxing a wildcard subscription onto SubjectSystemWildcard has no other +// check on what actually arrives. +func TestTaskIDFromSystemSubject(t *testing.T) { + tests := []struct { + name string + subject string + wantID int64 + wantOK bool + }{ + {"round trip with SubjectForSystem", SubjectForSystem(42), 42, true}, + {"round trip with a large ID", SubjectForSystem(9_223_372_036), 9_223_372_036, true}, + {"wrong kind suffix is rejected", "archie.agent.42.response", 0, false}, + {"missing prefix is rejected", "some.other.42.system", 0, false}, + {"non-numeric ID is rejected", "archie.agent.abc.system", 0, false}, + {"zero ID is rejected", "archie.agent.0.system", 0, false}, + {"negative ID is rejected", "archie.agent.-1.system", 0, false}, + {"empty subject is rejected", "", 0, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotID, gotOK := TaskIDFromSystemSubject(tt.subject) + if gotID != tt.wantID || gotOK != tt.wantOK { + t.Errorf("TaskIDFromSystemSubject(%q) = (%d, %v), want (%d, %v)", + tt.subject, gotID, gotOK, tt.wantID, tt.wantOK) + } + }) + } +} + +func TestSubjectSystemWildcardMatchesEverySystemSubject(t *testing.T) { + tests := []int64{1, 42, 9_223_372_036} + for _, taskID := range tests { + if id, ok := TaskIDFromSystemSubject(SubjectForSystem(taskID)); !ok || id != taskID { + t.Errorf("SubjectForSystem(%d) did not round-trip through TaskIDFromSystemSubject: got (%d, %v)", taskID, id, ok) + } + } + if SubjectSystemWildcard != "archie.agent.*.system" { + t.Errorf("SubjectSystemWildcard = %q, want the single-token wildcard NATS matches against subjectForTask's shape", + SubjectSystemWildcard) + } +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 9c3cecfeb..b20cd4a9c 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -24,6 +24,7 @@ import ( "github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/forge" + "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/memory" "github.com/samcharles93/archie-core/internal/plugin" "github.com/samcharles93/archie-core/internal/storage" @@ -120,6 +121,13 @@ type Daemon struct { // tool discovery (backward compatible). ToolRegistry *tools.Registry + // TaskLogs persists each task's own log output (including a sandboxed + // container's, which otherwise disappears at AutoRemove) and mirrors it + // live onto the dashboard feed, wired by the composition root. Nil + // disables task logging (backward compatible); every TaskRegistry + // method is nil-receiver-safe, so process() calls it unconditionally. + TaskLogs *logging.TaskRegistry + // running holds a cancel function for every task currently executing, // so /stop can reach work already in flight. Its zero value is ready // to use. @@ -757,6 +765,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { // this is the one place that has to do it. ctx, finished := d.running.begin(ctx, task.ID, task.Identity) defer finished() + defer d.openTaskLog(task)() fg := d.forgeFor(task) trees := d.treesFor(task) @@ -877,6 +886,24 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { } } +// openTaskLog opens task's log sink for the lifetime of one process() call +// and returns the function to close it, covering both dispatch paths below +// (sandboxed container and in-process) uniformly regardless of which one +// runs. A failure to open is not fatal to the task -- it means this +// attempt's own output won't be recoverable if something goes wrong, which +// is worse than the pre-existing state, not equal to it, so the run +// proceeds rather than parking over a logging problem. +func (d *Daemon) openTaskLog(task *store.Task) func() { + if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil { + d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err) + } + return func() { + if err := d.TaskLogs.Close(task.ID); err != nil { + d.Log.Warn("task log sink close failed", "task", task.ID, "err", err) + } + } +} + // acquireTaskContainer prepares and acquires the sandbox container for a // task, reporting false when it could not -- having already parked the // task and logged why. The caller releases the container it returns. diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 796fa0733..fb42af542 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -19,6 +19,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/forge" arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" + "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/secret" "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/store" @@ -66,6 +67,30 @@ func TestCleanupExpiredStorageUsesConfiguredTTL(t *testing.T) { } } +// TestOpenTaskLogOpensAndClosesTheRightSink guards process()'s wiring of +// TaskLogs without the ~10-dependency harness a full process() test would +// need (Store, Forge, Trees, ContainerPool, ...): openTaskLog only ever +// touches d.TaskLogs and d.Log, so it's testable in complete isolation. +// Uses TaskRegistry.Write's own true/false contract as the observable -- +// true right after opening (proving Open ran under this task's actual +// ID/attempt, not a typo'd constant), false after the returned closer runs +// (proving it's a real Close, not a no-op). +func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) { + reg := logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{}) + d := &Daemon{TaskLogs: reg, Log: slog.New(slog.DiscardHandler)} + task := &store.Task{ID: 5, Attempt: 2} + + closeFn := d.openTaskLog(task) + if ok := reg.Write(5, logging.Entry{Message: "x"}); !ok { + t.Fatal("Write() = false right after openTaskLog, want true (sink should be open)") + } + + closeFn() + if ok := reg.Write(5, logging.Entry{Message: "y"}); ok { + t.Error("Write() = true after openTaskLog's closer ran, want false (sink should be closed)") + } +} + func TestTaskDispatcherEnforcesMaxConcurrency(t *testing.T) { t.Parallel() diff --git a/internal/logging/registry.go b/internal/logging/registry.go new file mode 100644 index 000000000..407de9edc --- /dev/null +++ b/internal/logging/registry.go @@ -0,0 +1,122 @@ +package logging + +import ( + "context" + "log/slog" + "os" + "sync" +) + +// TaskRegistry manages one open log destination per currently-running task +// attempt, shared between whatever starts and ends a task's run (Open, +// Close) and whatever receives that task's remotely-shipped log records +// (Write) -- in the daemon, two different goroutines: task dispatch, and +// the NATS system-log subscription handler. +// +// The zero value is not usable; construct with NewTaskRegistry. A nil +// *TaskRegistry is safe to call every method on and always behaves as +// "task logging is not configured" -- the same "optional, nil means +// disabled" convention every other composition-wired field in Daemon +// follows, so callers never need a separate nil check before using one. +type TaskRegistry struct { + baseDir string + feed *Feed + opts TaskSinkOptions + + mu sync.Mutex + open map[int64]*registeredSink +} + +type registeredSink struct { + sink *TaskSink + logger *slog.Logger +} + +// NewTaskRegistry builds a registry writing under baseDir, mirroring every +// write into feed (may be nil to skip mirroring) the same way a locally +// logged entry would appear on the dashboard. +func NewTaskRegistry(baseDir string, feed *Feed, opts TaskSinkOptions) *TaskRegistry { + return &TaskRegistry{baseDir: baseDir, feed: feed, opts: opts, open: make(map[int64]*registeredSink)} +} + +// Open starts logging for a task attempt. A previous attempt already open +// under the same task ID is closed first: a retry reuses the task ID at a +// higher Attempt, and the old attempt's sink must stop receiving lines +// under its name rather than silently keep taking them. +func (r *TaskRegistry) Open(taskID int64, attempt int) error { + if r == nil { + return nil + } + sink, err := NewTaskSink(r.baseDir, taskID, attempt, r.opts) + if err != nil { + return err + } + logger := slog.New(NewFeedHandler(sink.Logger().Handler(), r.feed)) + + r.mu.Lock() + prev := r.open[taskID] + r.open[taskID] = ®isteredSink{sink: sink, logger: logger} + r.mu.Unlock() + + if prev != nil { + return prev.sink.Close() + } + return nil +} + +// Close stops logging for a task and releases its file handle. Safe to call +// for a task with no open sink -- every caller ending a task's run calls +// this unconditionally, whether or not Open ever succeeded for it. +func (r *TaskRegistry) Close(taskID int64) error { + if r == nil { + return nil + } + r.mu.Lock() + entry, ok := r.open[taskID] + delete(r.open, taskID) + r.mu.Unlock() + if !ok { + return nil + } + return entry.sink.Close() +} + +// Write appends one remotely-shipped entry to taskID's open sink, mirroring +// it into the live dashboard feed the same way a locally-written log would +// be. Reports false for a task with no open sink -- not an error: a system +// log message is fire-and-forget best effort, and a late or duplicate +// delivery after the task finished (or one this daemon instance never +// dispatched) is expected, not exceptional. +func (r *TaskRegistry) Write(taskID int64, entry Entry) bool { + if r == nil { + return false + } + r.mu.Lock() + got, ok := r.open[taskID] + r.mu.Unlock() + if !ok { + return false + } + + var level slog.Level + _ = level.UnmarshalText([]byte(entry.Level)) // unparseable falls back to Info, slog.Level's zero value + + attrs := make([]slog.Attr, 0, len(entry.Fields)) + for k, v := range entry.Fields { + attrs = append(attrs, slog.Any(k, v)) + } + got.logger.LogAttrs(context.Background(), level, entry.Message, attrs...) + return true +} + +// Remove deletes taskID's entire log directory -- every attempt, every +// rotated generation -- so task logs don't accumulate forever once a task +// is archived. Safe to call whether or not a sink is currently open (Close +// first regardless: this only removes files, it does not stop writes to an +// open sink) and whether or not the directory exists at all. +func (r *TaskRegistry) Remove(taskID int64) error { + if r == nil { + return nil + } + return os.RemoveAll(TaskLogDir(r.baseDir, taskID)) +} diff --git a/internal/logging/registry_test.go b/internal/logging/registry_test.go new file mode 100644 index 000000000..82e74a3a9 --- /dev/null +++ b/internal/logging/registry_test.go @@ -0,0 +1,191 @@ +package logging + +import ( + "os" + "sync" + "testing" +) + +func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) { + baseDir := t.TempDir() + feed := NewFeed(10) + reg := NewTaskRegistry(baseDir, feed, TaskSinkOptions{}) + + if err := reg.Open(42, 1); err != nil { + t.Fatalf("Open: %v", err) + } + if ok := reg.Write(42, Entry{Level: "WARN", Message: "gate failed", Fields: map[string]any{"component": "gate"}}); !ok { + t.Fatal("Write() = false, want true for an open task") + } + if err := reg.Close(42); err != nil { + t.Fatalf("Close: %v", err) + } + + result, err := Tail(TaskLogPath(baseDir, 42, 1), Query{}) + if err != nil { + t.Fatalf("Tail: %v", err) + } + if len(result.Entries) != 1 { + t.Fatalf("got %d entries, want 1: %+v", len(result.Entries), result.Entries) + } + entry := result.Entries[0] + if entry.Message != "gate failed" || entry.Level != "WARN" || entry.Fields["component"] != "gate" { + t.Errorf("entry = %+v, want message=gate failed level=WARN component=gate", entry) + } +} + +// TestTaskRegistryWriteMirrorsToTheDashboardFeed guards the other half of +// the point of routing writes through a slog.Logger built on FeedHandler +// instead of writing the file directly: a remotely-shipped entry must show +// up live on the dashboard the same way a locally-written one does. +func TestTaskRegistryWriteMirrorsToTheDashboardFeed(t *testing.T) { + baseDir := t.TempDir() + feed := NewFeed(10) + reg := NewTaskRegistry(baseDir, feed, TaskSinkOptions{}) + + if err := reg.Open(1, 1); err != nil { + t.Fatalf("Open: %v", err) + } + reg.Write(1, Entry{Level: "INFO", Message: "hello"}) + if err := reg.Close(1); err != nil { + t.Fatal(err) + } + + snapshot := feed.Snapshot() + if len(snapshot) != 1 || snapshot[0].Message != "hello" { + t.Fatalf("feed snapshot = %+v, want one entry with message %q", snapshot, "hello") + } +} + +// TestTaskRegistryWriteWithoutOpenIsANoOp pins that a system-log message for +// a task with no currently-open sink -- a late or duplicate delivery after +// the task finished, or one this daemon instance never dispatched -- is +// expected, not exceptional: no panic, no error return path to check. +func TestTaskRegistryWriteWithoutOpenIsANoOp(t *testing.T) { + reg := NewTaskRegistry(t.TempDir(), NewFeed(10), TaskSinkOptions{}) + if ok := reg.Write(999, Entry{Message: "orphan"}); ok { + t.Error("Write() = true for a task with no open sink, want false") + } +} + +// TestTaskRegistryOpenReplacesThePreviousAttempt guards a retry: the same +// task ID reopens at a higher Attempt, and writes after that must land in +// the new attempt's file, not silently keep flowing to the old one under +// its name. +func TestTaskRegistryOpenReplacesThePreviousAttempt(t *testing.T) { + baseDir := t.TempDir() + reg := NewTaskRegistry(baseDir, NewFeed(10), TaskSinkOptions{}) + + if err := reg.Open(7, 1); err != nil { + t.Fatalf("Open attempt 1: %v", err) + } + reg.Write(7, Entry{Message: "attempt one"}) + + if err := reg.Open(7, 2); err != nil { + t.Fatalf("Open attempt 2: %v", err) + } + reg.Write(7, Entry{Message: "attempt two"}) + if err := reg.Close(7); err != nil { + t.Fatal(err) + } + + attemptOne, err := Tail(TaskLogPath(baseDir, 7, 1), Query{}) + if err != nil { + t.Fatal(err) + } + if len(attemptOne.Entries) != 1 || attemptOne.Entries[0].Message != "attempt one" { + t.Errorf("attempt 1 file = %+v, want exactly [attempt one]", attemptOne.Entries) + } + + attemptTwo, err := Tail(TaskLogPath(baseDir, 7, 2), Query{}) + if err != nil { + t.Fatal(err) + } + if len(attemptTwo.Entries) != 1 || attemptTwo.Entries[0].Message != "attempt two" { + t.Errorf("attempt 2 file = %+v, want exactly [attempt two]", attemptTwo.Entries) + } +} + +func TestTaskRegistryCloseWithoutOpenIsSafe(t *testing.T) { + reg := NewTaskRegistry(t.TempDir(), NewFeed(10), TaskSinkOptions{}) + if err := reg.Close(123); err != nil { + t.Errorf("Close() on a never-opened task = %v, want nil", err) + } +} + +func TestTaskRegistryRemoveDeletesTheTaskDirectory(t *testing.T) { + baseDir := t.TempDir() + reg := NewTaskRegistry(baseDir, NewFeed(10), TaskSinkOptions{}) + + if err := reg.Open(5, 1); err != nil { + t.Fatal(err) + } + reg.Write(5, Entry{Message: "x"}) + if err := reg.Close(5); err != nil { + t.Fatal(err) + } + + dir := TaskLogDir(baseDir, 5) + if _, err := os.Stat(dir); err != nil { + t.Fatalf("task dir missing before Remove: %v", err) + } + if err := reg.Remove(5); err != nil { + t.Fatalf("Remove: %v", err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Errorf("task dir still present after Remove: err=%v", err) + } +} + +func TestTaskRegistryRemoveOnUnopenedTaskIsNotAnError(t *testing.T) { + reg := NewTaskRegistry(t.TempDir(), NewFeed(10), TaskSinkOptions{}) + if err := reg.Remove(404); err != nil { + t.Errorf("Remove() on a task with no directory = %v, want nil", err) + } +} + +// TestTaskRegistryNilReceiverIsSafe mirrors Feed's own nil-safety: a +// *TaskRegistry is wired by composition and left nil when task logging +// isn't configured, the same "optional, nil means disabled" convention +// every other Daemon field follows. Every method must tolerate that without +// the caller checking first. +func TestTaskRegistryNilReceiverIsSafe(t *testing.T) { + var reg *TaskRegistry + + if err := reg.Open(1, 1); err != nil { + t.Errorf("Open() on nil registry = %v, want nil", err) + } + if ok := reg.Write(1, Entry{Message: "x"}); ok { + t.Error("Write() on nil registry = true, want false") + } + if err := reg.Close(1); err != nil { + t.Errorf("Close() on nil registry = %v, want nil", err) + } + if err := reg.Remove(1); err != nil { + t.Errorf("Remove() on nil registry = %v, want nil", err) + } +} + +// TestTaskRegistryConcurrentTasksDoNotRace exercises the registry the way +// the daemon actually will: many tasks' Open/Write/Close interleaved from +// different goroutines (task dispatch vs. the NATS subscription handler), +// none sharing a task ID. Run with -race. +func TestTaskRegistryConcurrentTasksDoNotRace(t *testing.T) { + reg := NewTaskRegistry(t.TempDir(), NewFeed(100), TaskSinkOptions{}) + var wg sync.WaitGroup + for i := int64(1); i <= 20; i++ { + wg.Add(1) + go func(taskID int64) { + defer wg.Done() + if err := reg.Open(taskID, 1); err != nil { + t.Errorf("Open(%d): %v", taskID, err) + return + } + reg.Write(taskID, Entry{Message: "line"}) + if err := reg.Close(taskID); err != nil { + t.Errorf("Close(%d): %v", taskID, err) + } + }(i) + } + wg.Wait() +} diff --git a/internal/logging/tasksink.go b/internal/logging/tasksink.go index eee7cb98e..6a16c9a7f 100644 --- a/internal/logging/tasksink.go +++ b/internal/logging/tasksink.go @@ -7,12 +7,18 @@ import ( "strconv" ) -// TaskLogPath returns the log file path for one task attempt under baseDir -// (typically /logs/tasks). Each task gets its own subdirectory so -// an attempt's rotated generations (attempt-N.jsonl.1, .2, ...) stay grouped -// with that task rather than interleaved with every other task's. +// TaskLogDir returns the directory holding every attempt's log files for one +// task under baseDir (typically /logs/tasks). +func TaskLogDir(baseDir string, taskID int64) string { + return filepath.Join(baseDir, strconv.FormatInt(taskID, 10)) +} + +// TaskLogPath returns the log file path for one task attempt under baseDir. +// Each task gets its own subdirectory (TaskLogDir) so an attempt's rotated +// generations (attempt-N.jsonl.1, .2, ...) stay grouped with that task +// rather than interleaved with every other task's. func TaskLogPath(baseDir string, taskID int64, attempt int) string { - return filepath.Join(baseDir, strconv.FormatInt(taskID, 10), fmt.Sprintf("attempt-%d.jsonl", attempt)) + return filepath.Join(TaskLogDir(baseDir, taskID), fmt.Sprintf("attempt-%d.jsonl", attempt)) } // TaskSinkOptions configures a per-task log sink. The zero value selects the diff --git a/internal/webui/api_tasks.go b/internal/webui/api_tasks.go index af2299935..549d2060f 100644 --- a/internal/webui/api_tasks.go +++ b/internal/webui/api_tasks.go @@ -287,6 +287,23 @@ func (s *Server) archiveTask(ctx context.Context, w http.ResponseWriter, task *s if s.storeFailed(w, err) { return nil } + // Best-effort: an archived task's log files not getting cleaned up is + // far less costly than the archive action itself failing over it, so + // this never blocks the response the way s.storeFailed above does. + // + // Known, accepted gap: a task reaches a terminal status (satisfying + // ArchiveTask's precondition) partway through Daemon.process, but its + // TaskLogs sink isn't closed until process() itself returns -- which can + // be later, if teardown after the terminal transition takes any time at + // all. A system-log message landing in that window races this Remove + // with an open Write to the same directory. Not corrupting (the write + // lands on a soon-to-be-unlinked inode and is discarded) and not worth + // closing here: doing so would mean tying TaskLogs.Close to the store + // transition instead of to process()'s return, a larger change than + // this call site should make unilaterally. + if err := s.TaskLogs.Remove(task.ID); err != nil { + s.Log.Warn("task log cleanup failed", "task", task.ID, "err", err) + } return &actionOutcome{kind: events.KindTaskArchiveRequested, detail: detail, eventID: eventID} } diff --git a/internal/webui/api_tasks_actions_test.go b/internal/webui/api_tasks_actions_test.go index c6c4e2ff6..3b0057f71 100644 --- a/internal/webui/api_tasks_actions_test.go +++ b/internal/webui/api_tasks_actions_test.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" "slices" "strconv" "strings" @@ -15,6 +16,7 @@ import ( "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/store" ) @@ -542,6 +544,44 @@ func TestArchiveRemovesOnlyOneTerminalTask(t *testing.T) { } } +// TestArchiveRemovesTheTaskLogDirectory guards the cleanup half of task +// logging: without this, an archived task's log files (and every rotated +// generation under it) would accumulate on disk forever, since nothing else +// in the system ever visits a task again once it leaves the active board. +func TestArchiveRemovesTheTaskLogDirectory(t *testing.T) { + srv := newTestServer(t) + ctx := t.Context() + baseDir := t.TempDir() + srv.TaskLogs = logging.NewTaskRegistry(baseDir, logging.NewFeed(10), logging.TaskSinkOptions{}) + + if _, err := srv.Store.EnqueueIssue(ctx, "acme", "widget", 1, "done", "", "", ""); err != nil { + t.Fatal(err) + } + task, _ := srv.Store.TaskByIssue(ctx, "acme", "widget", 1) + if err := srv.Store.Transition(ctx, task.ID, store.StatusQueued, store.StatusMerged, "done"); err != nil { + t.Fatal(err) + } + + if err := srv.TaskLogs.Open(task.ID, task.Attempt); err != nil { + t.Fatal(err) + } + srv.TaskLogs.Write(task.ID, logging.Entry{Message: "line"}) + if err := srv.TaskLogs.Close(task.ID); err != nil { + t.Fatal(err) + } + dir := logging.TaskLogDir(baseDir, task.ID) + if _, err := os.Stat(dir); err != nil { + t.Fatalf("task log dir missing before archive: %v", err) + } + + if w := postAction(t, srv, task.ID, "archive"); w.Code != http.StatusOK { + t.Fatalf("archive status = %d, body %s", w.Code, w.Body) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Errorf("task log dir still present after archive: err=%v", err) + } +} + func TestArchiveAuditFailurePreservesTaskAndFailsRequest(t *testing.T) { srv := newTestServer(t) ctx := t.Context() diff --git a/internal/webui/server.go b/internal/webui/server.go index 8337f0184..e5c1ac927 100644 --- a/internal/webui/server.go +++ b/internal/webui/server.go @@ -43,6 +43,10 @@ type Server struct { // LogFeed is the daemon diagnostic stream. It is separate from Events, // which contains persisted task lifecycle activity only. LogFeed *logging.Feed + // TaskLogs holds each task's persisted log output. Archiving a task + // removes its log files here too, via TaskLogs.Remove -- nil (task + // logging not configured) makes that a no-op. + TaskLogs *logging.TaskRegistry // ConfigProvenance is supplied by composition from the production loader. // It is safe to expose: it contains paths and precedence only, never values.