Merge pull request #460 from samcharles93/fix/454-daemon-log-consumer
feat(daemon): consume agent system logs into per-task sinks and the dashboard feed
This commit is contained in:
commit
548f0601d3
11 files changed
+561
-8
No files matched your search
+47
-1
@@ -7,6 +7,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -308,6 +309,11 @@ func run() int {
|
||||
// daemon continues on stderr -- losing the durable copy must not take the
|
||||
// daemon down with it.
|
||||
logFeed := logging.NewFeed(1000)
|
||||
// Task logs live alongside the store rather than under cfg.Log.File's
|
||||
// directory: cfg.Log.File is optional (file logging can be off), while
|
||||
// DBPath is required for the daemon to run at all, so it is the more
|
||||
// reliable anchor for "where archie keeps its state" on this host.
|
||||
taskLogs := logging.NewTaskRegistry(filepath.Join(filepath.Dir(cfg.DBPath), "logs", "tasks"), logFeed, logging.TaskSinkOptions{})
|
||||
fileLog, logCloser, logErr := logging.New(logging.Options{
|
||||
File: cfg.Log.File,
|
||||
MaxSizeMB: cfg.Log.MaxSizeMB,
|
||||
@@ -390,7 +396,7 @@ func run() int {
|
||||
Path: origin.Path, Role: string(origin.Role), Layer: string(origin.Layer), Feature: string(origin.Feature),
|
||||
})
|
||||
}
|
||||
web := &webui.Server{Store: st, Log: log, LogFeed: logFeed, Cfg: &cfg, ConfigProvenance: configProvenance, Channels: channelManager, Events: bus}
|
||||
web := &webui.Server{Store: st, Log: log, LogFeed: logFeed, TaskLogs: taskLogs, Cfg: &cfg, ConfigProvenance: configProvenance, Channels: channelManager, Events: bus}
|
||||
web.ReloadChannel = func(ctx context.Context, id string) error {
|
||||
if id != "telegram" || restartTelegram == nil {
|
||||
return fmt.Errorf("channel reload unavailable")
|
||||
@@ -717,6 +723,13 @@ func run() int {
|
||||
return 1
|
||||
}
|
||||
defer unsubscribe()
|
||||
|
||||
unsubscribeSystemLogs, err := subscribeSystemLogs(coreConn, taskLogs, log)
|
||||
if err != nil {
|
||||
log.Error("system log subscribe failed", "err", err)
|
||||
return 1
|
||||
}
|
||||
defer unsubscribeSystemLogs()
|
||||
}
|
||||
|
||||
// ── Memory manager ────────────────────────────────────────────────
|
||||
@@ -917,6 +930,7 @@ func run() int {
|
||||
ToolRegistry: toolReg,
|
||||
Curators: curatorRegistry,
|
||||
Identities: identityRunners,
|
||||
TaskLogs: taskLogs,
|
||||
}
|
||||
web.TaskStopper = d
|
||||
|
||||
@@ -1128,6 +1142,38 @@ func executionProviders(cfg config.Config) map[string]agentexec.Provider {
|
||||
return agentexec.ProvidersFromConfig(cfg.Providers)
|
||||
}
|
||||
|
||||
// subscribeSystemLogs subscribes to every task's system log subject at once.
|
||||
// A sandboxed container's own stderr disappears at AutoRemove; archie-agent
|
||||
// ships it here instead (agentexec.SubjectForSystem), and this is where it
|
||||
// lands: one wildcard subscription demuxed by taskID and routed through
|
||||
// taskLogs, which is opened for a task's duration in Daemon.process, so a
|
||||
// message for a task not currently open there -- late, duplicate, or from
|
||||
// an attempt this instance never dispatched -- is expected and silently
|
||||
// dropped rather than treated as an error.
|
||||
func subscribeSystemLogs(nc *natsio.Conn, taskLogs *logging.TaskRegistry, log *slog.Logger) (unsubscribe func(), err error) {
|
||||
sub, err := nc.Subscribe(agentexec.SubjectSystemWildcard, func(msg *natsio.Msg) {
|
||||
taskID, ok := agentexec.TaskIDFromSystemSubject(msg.Subject)
|
||||
if !ok {
|
||||
log.Warn("system log message on unparseable subject", "subject", msg.Subject)
|
||||
return
|
||||
}
|
||||
var entry logging.Entry
|
||||
if err := json.Unmarshal(msg.Data, &entry); err != nil {
|
||||
log.Warn("system log message undecodable", "task", taskID, "err", err)
|
||||
return
|
||||
}
|
||||
taskLogs.Write(taskID, entry)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() {
|
||||
if err := sub.Unsubscribe(); err != nil {
|
||||
log.Warn("system log unsubscribe failed", "err", err)
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// registerTaskRPCServers subscribes the storerpc/forgerpc/worktreerpc
|
||||
// handlers on nc so an archie-agent container (which holds no DB
|
||||
// connection, forge token, or push credential) can proxy those operations
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package agentexec
|
||||
|
||||
import "strconv"
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Agent execution subjects.
|
||||
//
|
||||
@@ -16,7 +19,13 @@ const (
|
||||
// SubjectAgentWildcard matches every agent subject.
|
||||
SubjectAgentWildcard = "archie.agent.>"
|
||||
|
||||
subjectAgentPrefix = "archie.agent."
|
||||
// SubjectSystemWildcard matches every task's system subject, for a
|
||||
// daemon subscribing to all of them at once with a single core NATS
|
||||
// subscription rather than one per running task.
|
||||
SubjectSystemWildcard = subjectAgentPrefix + "*.system"
|
||||
|
||||
subjectAgentPrefix = "archie.agent."
|
||||
subjectSystemSuffix = ".system"
|
||||
)
|
||||
|
||||
// SubjectForRequest returns the subject carrying stage execution requests for
|
||||
@@ -43,3 +52,20 @@ func SubjectForSystem(taskID int64) string {
|
||||
func subjectForTask(taskID int64, kind string) string {
|
||||
return subjectAgentPrefix + strconv.FormatInt(taskID, 10) + "." + kind
|
||||
}
|
||||
|
||||
// TaskIDFromSystemSubject extracts the task ID from a subject produced by
|
||||
// SubjectForSystem, for a daemon demuxing a SubjectSystemWildcard
|
||||
// subscription. Rejects (0, false) for anything that isn't exactly that
|
||||
// shape, rather than parsing a partial or wrong-kind subject into a
|
||||
// misleading task ID.
|
||||
func TaskIDFromSystemSubject(subject string) (int64, bool) {
|
||||
if !strings.HasPrefix(subject, subjectAgentPrefix) || !strings.HasSuffix(subject, subjectSystemSuffix) {
|
||||
return 0, false
|
||||
}
|
||||
middle := strings.TrimSuffix(strings.TrimPrefix(subject, subjectAgentPrefix), subjectSystemSuffix)
|
||||
id, err := strconv.ParseInt(middle, 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package agentexec
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestTaskIDFromSystemSubject pins the round trip with SubjectForSystem, and
|
||||
// that a subject which merely resembles one (wrong kind suffix, malformed
|
||||
// ID) is rejected rather than parsed into a wrong task ID -- a daemon
|
||||
// demuxing a wildcard subscription onto SubjectSystemWildcard has no other
|
||||
// check on what actually arrives.
|
||||
func TestTaskIDFromSystemSubject(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
subject string
|
||||
wantID int64
|
||||
wantOK bool
|
||||
}{
|
||||
{"round trip with SubjectForSystem", SubjectForSystem(42), 42, true},
|
||||
{"round trip with a large ID", SubjectForSystem(9_223_372_036), 9_223_372_036, true},
|
||||
{"wrong kind suffix is rejected", "archie.agent.42.response", 0, false},
|
||||
{"missing prefix is rejected", "some.other.42.system", 0, false},
|
||||
{"non-numeric ID is rejected", "archie.agent.abc.system", 0, false},
|
||||
{"zero ID is rejected", "archie.agent.0.system", 0, false},
|
||||
{"negative ID is rejected", "archie.agent.-1.system", 0, false},
|
||||
{"empty subject is rejected", "", 0, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotID, gotOK := TaskIDFromSystemSubject(tt.subject)
|
||||
if gotID != tt.wantID || gotOK != tt.wantOK {
|
||||
t.Errorf("TaskIDFromSystemSubject(%q) = (%d, %v), want (%d, %v)",
|
||||
tt.subject, gotID, gotOK, tt.wantID, tt.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubjectSystemWildcardMatchesEverySystemSubject(t *testing.T) {
|
||||
tests := []int64{1, 42, 9_223_372_036}
|
||||
for _, taskID := range tests {
|
||||
if id, ok := TaskIDFromSystemSubject(SubjectForSystem(taskID)); !ok || id != taskID {
|
||||
t.Errorf("SubjectForSystem(%d) did not round-trip through TaskIDFromSystemSubject: got (%d, %v)", taskID, id, ok)
|
||||
}
|
||||
}
|
||||
if SubjectSystemWildcard != "archie.agent.*.system" {
|
||||
t.Errorf("SubjectSystemWildcard = %q, want the single-token wildcard NATS matches against subjectForTask's shape",
|
||||
SubjectSystemWildcard)
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
"github.com/samcharles93/archie-core/internal/memory"
|
||||
"github.com/samcharles93/archie-core/internal/plugin"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
@@ -120,6 +121,13 @@ type Daemon struct {
|
||||
// tool discovery (backward compatible).
|
||||
ToolRegistry *tools.Registry
|
||||
|
||||
// TaskLogs persists each task's own log output (including a sandboxed
|
||||
// container's, which otherwise disappears at AutoRemove) and mirrors it
|
||||
// live onto the dashboard feed, wired by the composition root. Nil
|
||||
// disables task logging (backward compatible); every TaskRegistry
|
||||
// method is nil-receiver-safe, so process() calls it unconditionally.
|
||||
TaskLogs *logging.TaskRegistry
|
||||
|
||||
// running holds a cancel function for every task currently executing,
|
||||
// so /stop can reach work already in flight. Its zero value is ready
|
||||
// to use.
|
||||
@@ -757,6 +765,7 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
|
||||
// this is the one place that has to do it.
|
||||
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
||||
defer finished()
|
||||
defer d.openTaskLog(task)()
|
||||
|
||||
fg := d.forgeFor(task)
|
||||
trees := d.treesFor(task)
|
||||
@@ -877,6 +886,24 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
|
||||
}
|
||||
}
|
||||
|
||||
// openTaskLog opens task's log sink for the lifetime of one process() call
|
||||
// and returns the function to close it, covering both dispatch paths below
|
||||
// (sandboxed container and in-process) uniformly regardless of which one
|
||||
// runs. A failure to open is not fatal to the task -- it means this
|
||||
// attempt's own output won't be recoverable if something goes wrong, which
|
||||
// is worse than the pre-existing state, not equal to it, so the run
|
||||
// proceeds rather than parking over a logging problem.
|
||||
func (d *Daemon) openTaskLog(task *store.Task) func() {
|
||||
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
|
||||
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
|
||||
}
|
||||
return func() {
|
||||
if err := d.TaskLogs.Close(task.ID); err != nil {
|
||||
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// acquireTaskContainer prepares and acquires the sandbox container for a
|
||||
// task, reporting false when it could not -- having already parked the
|
||||
// task and logged why. The caller releases the container it returns.
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
@@ -66,6 +67,30 @@ func TestCleanupExpiredStorageUsesConfiguredTTL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenTaskLogOpensAndClosesTheRightSink guards process()'s wiring of
|
||||
// TaskLogs without the ~10-dependency harness a full process() test would
|
||||
// need (Store, Forge, Trees, ContainerPool, ...): openTaskLog only ever
|
||||
// touches d.TaskLogs and d.Log, so it's testable in complete isolation.
|
||||
// Uses TaskRegistry.Write's own true/false contract as the observable --
|
||||
// true right after opening (proving Open ran under this task's actual
|
||||
// ID/attempt, not a typo'd constant), false after the returned closer runs
|
||||
// (proving it's a real Close, not a no-op).
|
||||
func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) {
|
||||
reg := logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{})
|
||||
d := &Daemon{TaskLogs: reg, Log: slog.New(slog.DiscardHandler)}
|
||||
task := &store.Task{ID: 5, Attempt: 2}
|
||||
|
||||
closeFn := d.openTaskLog(task)
|
||||
if ok := reg.Write(5, logging.Entry{Message: "x"}); !ok {
|
||||
t.Fatal("Write() = false right after openTaskLog, want true (sink should be open)")
|
||||
}
|
||||
|
||||
closeFn()
|
||||
if ok := reg.Write(5, logging.Entry{Message: "y"}); ok {
|
||||
t.Error("Write() = true after openTaskLog's closer ran, want false (sink should be closed)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskDispatcherEnforcesMaxConcurrency(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// TaskRegistry manages one open log destination per currently-running task
|
||||
// attempt, shared between whatever starts and ends a task's run (Open,
|
||||
// Close) and whatever receives that task's remotely-shipped log records
|
||||
// (Write) -- in the daemon, two different goroutines: task dispatch, and
|
||||
// the NATS system-log subscription handler.
|
||||
//
|
||||
// The zero value is not usable; construct with NewTaskRegistry. A nil
|
||||
// *TaskRegistry is safe to call every method on and always behaves as
|
||||
// "task logging is not configured" -- the same "optional, nil means
|
||||
// disabled" convention every other composition-wired field in Daemon
|
||||
// follows, so callers never need a separate nil check before using one.
|
||||
type TaskRegistry struct {
|
||||
baseDir string
|
||||
feed *Feed
|
||||
opts TaskSinkOptions
|
||||
|
||||
mu sync.Mutex
|
||||
open map[int64]*registeredSink
|
||||
}
|
||||
|
||||
type registeredSink struct {
|
||||
sink *TaskSink
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewTaskRegistry builds a registry writing under baseDir, mirroring every
|
||||
// write into feed (may be nil to skip mirroring) the same way a locally
|
||||
// logged entry would appear on the dashboard.
|
||||
func NewTaskRegistry(baseDir string, feed *Feed, opts TaskSinkOptions) *TaskRegistry {
|
||||
return &TaskRegistry{baseDir: baseDir, feed: feed, opts: opts, open: make(map[int64]*registeredSink)}
|
||||
}
|
||||
|
||||
// Open starts logging for a task attempt. A previous attempt already open
|
||||
// under the same task ID is closed first: a retry reuses the task ID at a
|
||||
// higher Attempt, and the old attempt's sink must stop receiving lines
|
||||
// under its name rather than silently keep taking them.
|
||||
func (r *TaskRegistry) Open(taskID int64, attempt int) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
sink, err := NewTaskSink(r.baseDir, taskID, attempt, r.opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger := slog.New(NewFeedHandler(sink.Logger().Handler(), r.feed))
|
||||
|
||||
r.mu.Lock()
|
||||
prev := r.open[taskID]
|
||||
r.open[taskID] = ®isteredSink{sink: sink, logger: logger}
|
||||
r.mu.Unlock()
|
||||
|
||||
if prev != nil {
|
||||
return prev.sink.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close stops logging for a task and releases its file handle. Safe to call
|
||||
// for a task with no open sink -- every caller ending a task's run calls
|
||||
// this unconditionally, whether or not Open ever succeeded for it.
|
||||
func (r *TaskRegistry) Close(taskID int64) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
entry, ok := r.open[taskID]
|
||||
delete(r.open, taskID)
|
||||
r.mu.Unlock()
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return entry.sink.Close()
|
||||
}
|
||||
|
||||
// Write appends one remotely-shipped entry to taskID's open sink, mirroring
|
||||
// it into the live dashboard feed the same way a locally-written log would
|
||||
// be. Reports false for a task with no open sink -- not an error: a system
|
||||
// log message is fire-and-forget best effort, and a late or duplicate
|
||||
// delivery after the task finished (or one this daemon instance never
|
||||
// dispatched) is expected, not exceptional.
|
||||
func (r *TaskRegistry) Write(taskID int64, entry Entry) bool {
|
||||
if r == nil {
|
||||
return false
|
||||
}
|
||||
r.mu.Lock()
|
||||
got, ok := r.open[taskID]
|
||||
r.mu.Unlock()
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
var level slog.Level
|
||||
_ = level.UnmarshalText([]byte(entry.Level)) // unparseable falls back to Info, slog.Level's zero value
|
||||
|
||||
attrs := make([]slog.Attr, 0, len(entry.Fields))
|
||||
for k, v := range entry.Fields {
|
||||
attrs = append(attrs, slog.Any(k, v))
|
||||
}
|
||||
got.logger.LogAttrs(context.Background(), level, entry.Message, attrs...)
|
||||
return true
|
||||
}
|
||||
|
||||
// Remove deletes taskID's entire log directory -- every attempt, every
|
||||
// rotated generation -- so task logs don't accumulate forever once a task
|
||||
// is archived. Safe to call whether or not a sink is currently open (Close
|
||||
// first regardless: this only removes files, it does not stop writes to an
|
||||
// open sink) and whether or not the directory exists at all.
|
||||
func (r *TaskRegistry) Remove(taskID int64) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
return os.RemoveAll(TaskLogDir(r.baseDir, taskID))
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) {
|
||||
baseDir := t.TempDir()
|
||||
feed := NewFeed(10)
|
||||
reg := NewTaskRegistry(baseDir, feed, TaskSinkOptions{})
|
||||
|
||||
if err := reg.Open(42, 1); err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
if ok := reg.Write(42, Entry{Level: "WARN", Message: "gate failed", Fields: map[string]any{"component": "gate"}}); !ok {
|
||||
t.Fatal("Write() = false, want true for an open task")
|
||||
}
|
||||
if err := reg.Close(42); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
result, err := Tail(TaskLogPath(baseDir, 42, 1), Query{})
|
||||
if err != nil {
|
||||
t.Fatalf("Tail: %v", err)
|
||||
}
|
||||
if len(result.Entries) != 1 {
|
||||
t.Fatalf("got %d entries, want 1: %+v", len(result.Entries), result.Entries)
|
||||
}
|
||||
entry := result.Entries[0]
|
||||
if entry.Message != "gate failed" || entry.Level != "WARN" || entry.Fields["component"] != "gate" {
|
||||
t.Errorf("entry = %+v, want message=gate failed level=WARN component=gate", entry)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskRegistryWriteMirrorsToTheDashboardFeed guards the other half of
|
||||
// the point of routing writes through a slog.Logger built on FeedHandler
|
||||
// instead of writing the file directly: a remotely-shipped entry must show
|
||||
// up live on the dashboard the same way a locally-written one does.
|
||||
func TestTaskRegistryWriteMirrorsToTheDashboardFeed(t *testing.T) {
|
||||
baseDir := t.TempDir()
|
||||
feed := NewFeed(10)
|
||||
reg := NewTaskRegistry(baseDir, feed, TaskSinkOptions{})
|
||||
|
||||
if err := reg.Open(1, 1); err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
reg.Write(1, Entry{Level: "INFO", Message: "hello"})
|
||||
if err := reg.Close(1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
snapshot := feed.Snapshot()
|
||||
if len(snapshot) != 1 || snapshot[0].Message != "hello" {
|
||||
t.Fatalf("feed snapshot = %+v, want one entry with message %q", snapshot, "hello")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskRegistryWriteWithoutOpenIsANoOp pins that a system-log message for
|
||||
// a task with no currently-open sink -- a late or duplicate delivery after
|
||||
// the task finished, or one this daemon instance never dispatched -- is
|
||||
// expected, not exceptional: no panic, no error return path to check.
|
||||
func TestTaskRegistryWriteWithoutOpenIsANoOp(t *testing.T) {
|
||||
reg := NewTaskRegistry(t.TempDir(), NewFeed(10), TaskSinkOptions{})
|
||||
if ok := reg.Write(999, Entry{Message: "orphan"}); ok {
|
||||
t.Error("Write() = true for a task with no open sink, want false")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskRegistryOpenReplacesThePreviousAttempt guards a retry: the same
|
||||
// task ID reopens at a higher Attempt, and writes after that must land in
|
||||
// the new attempt's file, not silently keep flowing to the old one under
|
||||
// its name.
|
||||
func TestTaskRegistryOpenReplacesThePreviousAttempt(t *testing.T) {
|
||||
baseDir := t.TempDir()
|
||||
reg := NewTaskRegistry(baseDir, NewFeed(10), TaskSinkOptions{})
|
||||
|
||||
if err := reg.Open(7, 1); err != nil {
|
||||
t.Fatalf("Open attempt 1: %v", err)
|
||||
}
|
||||
reg.Write(7, Entry{Message: "attempt one"})
|
||||
|
||||
if err := reg.Open(7, 2); err != nil {
|
||||
t.Fatalf("Open attempt 2: %v", err)
|
||||
}
|
||||
reg.Write(7, Entry{Message: "attempt two"})
|
||||
if err := reg.Close(7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
attemptOne, err := Tail(TaskLogPath(baseDir, 7, 1), Query{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(attemptOne.Entries) != 1 || attemptOne.Entries[0].Message != "attempt one" {
|
||||
t.Errorf("attempt 1 file = %+v, want exactly [attempt one]", attemptOne.Entries)
|
||||
}
|
||||
|
||||
attemptTwo, err := Tail(TaskLogPath(baseDir, 7, 2), Query{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(attemptTwo.Entries) != 1 || attemptTwo.Entries[0].Message != "attempt two" {
|
||||
t.Errorf("attempt 2 file = %+v, want exactly [attempt two]", attemptTwo.Entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskRegistryCloseWithoutOpenIsSafe(t *testing.T) {
|
||||
reg := NewTaskRegistry(t.TempDir(), NewFeed(10), TaskSinkOptions{})
|
||||
if err := reg.Close(123); err != nil {
|
||||
t.Errorf("Close() on a never-opened task = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskRegistryRemoveDeletesTheTaskDirectory(t *testing.T) {
|
||||
baseDir := t.TempDir()
|
||||
reg := NewTaskRegistry(baseDir, NewFeed(10), TaskSinkOptions{})
|
||||
|
||||
if err := reg.Open(5, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg.Write(5, Entry{Message: "x"})
|
||||
if err := reg.Close(5); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := TaskLogDir(baseDir, 5)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
t.Fatalf("task dir missing before Remove: %v", err)
|
||||
}
|
||||
if err := reg.Remove(5); err != nil {
|
||||
t.Fatalf("Remove: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Errorf("task dir still present after Remove: err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskRegistryRemoveOnUnopenedTaskIsNotAnError(t *testing.T) {
|
||||
reg := NewTaskRegistry(t.TempDir(), NewFeed(10), TaskSinkOptions{})
|
||||
if err := reg.Remove(404); err != nil {
|
||||
t.Errorf("Remove() on a task with no directory = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskRegistryNilReceiverIsSafe mirrors Feed's own nil-safety: a
|
||||
// *TaskRegistry is wired by composition and left nil when task logging
|
||||
// isn't configured, the same "optional, nil means disabled" convention
|
||||
// every other Daemon field follows. Every method must tolerate that without
|
||||
// the caller checking first.
|
||||
func TestTaskRegistryNilReceiverIsSafe(t *testing.T) {
|
||||
var reg *TaskRegistry
|
||||
|
||||
if err := reg.Open(1, 1); err != nil {
|
||||
t.Errorf("Open() on nil registry = %v, want nil", err)
|
||||
}
|
||||
if ok := reg.Write(1, Entry{Message: "x"}); ok {
|
||||
t.Error("Write() on nil registry = true, want false")
|
||||
}
|
||||
if err := reg.Close(1); err != nil {
|
||||
t.Errorf("Close() on nil registry = %v, want nil", err)
|
||||
}
|
||||
if err := reg.Remove(1); err != nil {
|
||||
t.Errorf("Remove() on nil registry = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaskRegistryConcurrentTasksDoNotRace exercises the registry the way
|
||||
// the daemon actually will: many tasks' Open/Write/Close interleaved from
|
||||
// different goroutines (task dispatch vs. the NATS subscription handler),
|
||||
// none sharing a task ID. Run with -race.
|
||||
func TestTaskRegistryConcurrentTasksDoNotRace(t *testing.T) {
|
||||
reg := NewTaskRegistry(t.TempDir(), NewFeed(100), TaskSinkOptions{})
|
||||
var wg sync.WaitGroup
|
||||
for i := int64(1); i <= 20; i++ {
|
||||
wg.Add(1)
|
||||
go func(taskID int64) {
|
||||
defer wg.Done()
|
||||
if err := reg.Open(taskID, 1); err != nil {
|
||||
t.Errorf("Open(%d): %v", taskID, err)
|
||||
return
|
||||
}
|
||||
reg.Write(taskID, Entry{Message: "line"})
|
||||
if err := reg.Close(taskID); err != nil {
|
||||
t.Errorf("Close(%d): %v", taskID, err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -7,12 +7,18 @@ import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// TaskLogPath returns the log file path for one task attempt under baseDir
|
||||
// (typically <state_dir>/logs/tasks). Each task gets its own subdirectory so
|
||||
// an attempt's rotated generations (attempt-N.jsonl.1, .2, ...) stay grouped
|
||||
// with that task rather than interleaved with every other task's.
|
||||
// TaskLogDir returns the directory holding every attempt's log files for one
|
||||
// task under baseDir (typically <state_dir>/logs/tasks).
|
||||
func TaskLogDir(baseDir string, taskID int64) string {
|
||||
return filepath.Join(baseDir, strconv.FormatInt(taskID, 10))
|
||||
}
|
||||
|
||||
// TaskLogPath returns the log file path for one task attempt under baseDir.
|
||||
// Each task gets its own subdirectory (TaskLogDir) so an attempt's rotated
|
||||
// generations (attempt-N.jsonl.1, .2, ...) stay grouped with that task
|
||||
// rather than interleaved with every other task's.
|
||||
func TaskLogPath(baseDir string, taskID int64, attempt int) string {
|
||||
return filepath.Join(baseDir, strconv.FormatInt(taskID, 10), fmt.Sprintf("attempt-%d.jsonl", attempt))
|
||||
return filepath.Join(TaskLogDir(baseDir, taskID), fmt.Sprintf("attempt-%d.jsonl", attempt))
|
||||
}
|
||||
|
||||
// TaskSinkOptions configures a per-task log sink. The zero value selects the
|
||||
|
||||
@@ -287,6 +287,23 @@ func (s *Server) archiveTask(ctx context.Context, w http.ResponseWriter, task *s
|
||||
if s.storeFailed(w, err) {
|
||||
return nil
|
||||
}
|
||||
// Best-effort: an archived task's log files not getting cleaned up is
|
||||
// far less costly than the archive action itself failing over it, so
|
||||
// this never blocks the response the way s.storeFailed above does.
|
||||
//
|
||||
// Known, accepted gap: a task reaches a terminal status (satisfying
|
||||
// ArchiveTask's precondition) partway through Daemon.process, but its
|
||||
// TaskLogs sink isn't closed until process() itself returns -- which can
|
||||
// be later, if teardown after the terminal transition takes any time at
|
||||
// all. A system-log message landing in that window races this Remove
|
||||
// with an open Write to the same directory. Not corrupting (the write
|
||||
// lands on a soon-to-be-unlinked inode and is discarded) and not worth
|
||||
// closing here: doing so would mean tying TaskLogs.Close to the store
|
||||
// transition instead of to process()'s return, a larger change than
|
||||
// this call site should make unilaterally.
|
||||
if err := s.TaskLogs.Remove(task.ID); err != nil {
|
||||
s.Log.Warn("task log cleanup failed", "task", task.ID, "err", err)
|
||||
}
|
||||
return &actionOutcome{kind: events.KindTaskArchiveRequested, detail: detail, eventID: eventID}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
@@ -542,6 +544,44 @@ func TestArchiveRemovesOnlyOneTerminalTask(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestArchiveRemovesTheTaskLogDirectory guards the cleanup half of task
|
||||
// logging: without this, an archived task's log files (and every rotated
|
||||
// generation under it) would accumulate on disk forever, since nothing else
|
||||
// in the system ever visits a task again once it leaves the active board.
|
||||
func TestArchiveRemovesTheTaskLogDirectory(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
ctx := t.Context()
|
||||
baseDir := t.TempDir()
|
||||
srv.TaskLogs = logging.NewTaskRegistry(baseDir, logging.NewFeed(10), logging.TaskSinkOptions{})
|
||||
|
||||
if _, err := srv.Store.EnqueueIssue(ctx, "acme", "widget", 1, "done", "", "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, _ := srv.Store.TaskByIssue(ctx, "acme", "widget", 1)
|
||||
if err := srv.Store.Transition(ctx, task.ID, store.StatusQueued, store.StatusMerged, "done"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := srv.TaskLogs.Open(task.ID, task.Attempt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.TaskLogs.Write(task.ID, logging.Entry{Message: "line"})
|
||||
if err := srv.TaskLogs.Close(task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := logging.TaskLogDir(baseDir, task.ID)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
t.Fatalf("task log dir missing before archive: %v", err)
|
||||
}
|
||||
|
||||
if w := postAction(t, srv, task.ID, "archive"); w.Code != http.StatusOK {
|
||||
t.Fatalf("archive status = %d, body %s", w.Code, w.Body)
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Errorf("task log dir still present after archive: err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveAuditFailurePreservesTaskAndFailsRequest(t *testing.T) {
|
||||
srv := newTestServer(t)
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -43,6 +43,10 @@ type Server struct {
|
||||
// LogFeed is the daemon diagnostic stream. It is separate from Events,
|
||||
// which contains persisted task lifecycle activity only.
|
||||
LogFeed *logging.Feed
|
||||
// TaskLogs holds each task's persisted log output. Archiving a task
|
||||
// removes its log files here too, via TaskLogs.Remove -- nil (task
|
||||
// logging not configured) makes that a no-op.
|
||||
TaskLogs *logging.TaskRegistry
|
||||
|
||||
// ConfigProvenance is supplied by composition from the production loader.
|
||||
// It is safe to expose: it contains paths and precedence only, never values.
|
||||
|
||||
Reference in new issue
Block a user