diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 9e43277cf..e4d419353 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -1038,6 +1038,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) { d.Log.Warn("managed worker park transition failed", "task", task.ID, "err", err) return } + d.recordPark(ctx, task.ID, reason) d.emit(events.Event{ Kind: events.KindParked, TaskID: task.ID, Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason, @@ -1051,6 +1052,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) { d.Log.Warn("agent task transport park transition failed", "task", task.ID, "err", err) return } + d.recordPark(ctx, task.ID, reason) d.emit(events.Event{ Kind: events.KindParked, TaskID: task.ID, Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason, @@ -1263,7 +1265,33 @@ func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason strin if !errors.Is(err, storecontract.ErrStaleTransition) { d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err) } + return } + // Recorded only once the transition landed. An entry claiming the task + // parked would be a lie when another actor moved it out of running first + // -- the same race the parked event is withheld for. + d.recordPark(ctx, taskID, reason) +} + +// recordPark writes one park reason into the task attempt's own log. Every +// place that parks a task calls it, including the two capability parks in +// process() that write their transition directly instead of going through +// parkRunningTask -- a park an operator cannot read the reason for is the same +// defect wherever it happens. Nothing daemon-side used to write to a task log +// at all: the container's system-log subscription was the only writer, so an +// attempt that parked before any container produced output left an empty log +// file and an unexplained park reason. +// +// A task with no open sink (a retry this instance never dispatched, or logging +// unwired) silently drops the entry, which is TaskRegistry.Write's own +// contract. +func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) { + d.TaskLogs.Write(ctx, taskID, logging.Entry{ + Time: time.Now(), + Level: slog.LevelError.String(), + Message: "task parked: " + reason, + Fields: map[string]any{"component": "daemon", "task": taskID}, + }) } // containerEnv returns the environment variables passed to agent containers. diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index a83b3787f..617e606ca 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -80,16 +80,100 @@ func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) { task := &workflow.Task{ID: 5, Attempt: 2} closeFn := d.openTaskLog(task) - if ok := reg.Write(5, logging.Entry{Message: "x"}); !ok { + if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "x"}); !ok { t.Fatal("Write() = false right after openTaskLog, want true (sink should be open)") } closeFn() - if ok := reg.Write(5, logging.Entry{Message: "y"}); ok { + if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "y"}); ok { t.Error("Write() = true after openTaskLog's closer ran, want false (sink should be closed)") } } +// TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog pins the answer to the +// question the task log exists for: "why did this attempt park?" Every park +// used to be written only to the daemon-wide log, so an attempt that parked +// before its container produced any output -- a worktree prepare failure, an +// unavailable worker, a taskrun error -- left a task log that the sink had +// created and nothing had written to. The dashboard offered that zero-byte +// file as a download, and the reason lived somewhere the operator could not +// hand to anyone. +func TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog(t *testing.T) { + d, st, _ := testDaemon(t, 3, 0) + d.Log = slog.New(slog.DiscardHandler) + d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{}) + + ctx := t.Context() + if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { + t.Fatal(err) + } + task, err := st.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("ClaimNext = (%+v, %v)", task, err) + } + // The sink is opened for the attempt by process() and closed when the run + // ends; the park happens in between, which is the only window it is read in. + closeLog := d.openTaskLog(task) + defer closeLog() + + const reason = "worktree prepare failed: checkout branch fix/42-x: reference not found" + d.parkRunningTask(ctx, task.ID, reason) + + page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{}) + if err != nil { + t.Fatalf("TaskLog: %v", err) + } + if !page.Found { + t.Fatalf("attempt %d has no log file, want the reason recorded in one", task.Attempt) + } + if len(page.Entries) != 1 { + t.Fatalf("entries = %+v, want exactly the park reason", page.Entries) + } + entry := page.Entries[0] + if !strings.Contains(entry.Message, reason) { + t.Errorf("entry = %q, want it to carry the park reason %q", entry.Message, reason) + } + if entry.Level != slog.LevelError.String() { + t.Errorf("entry level = %q, want %q", entry.Level, slog.LevelError.String()) + } + if entry.Fields["component"] != "daemon" { + t.Errorf("entry component = %v, want the daemon to be identifiable in the attempt's own log", entry.Fields["component"]) + } +} + +// TestParkDoesNotRecordAParkThatLosesTheRace keeps the new task-log entry as +// truthful as the parked event it accompanies. If another actor moved the task +// out of running first, the guarded transition reports ErStoreStaleTransition +// and nothing was parked -- so "task parked: " must not appear in the +// attempt's log either, or the log claims an outcome the task board contradicts. +func TestParkDoesNotRecordAParkThatLosesTheRace(t *testing.T) { + d, st, _ := testDaemon(t, 3, 0) + d.Log = slog.New(slog.DiscardHandler) + d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{}) + + ctx := t.Context() + if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { + t.Fatal(err) + } + // Left queued, not claimed: the running -> parked transition cannot match. + task, err := st.TaskByIssue(ctx, "acme", "widget", 42) + if err != nil || task == nil { + t.Fatalf("TaskByIssue = (%+v, %v)", task, err) + } + closeLog := d.openTaskLog(task) + defer closeLog() + + d.parkRunningTask(ctx, task.ID, "worktree prepare failed: simulated") + + page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{}) + if err != nil { + t.Fatalf("TaskLog: %v", err) + } + if len(page.Entries) != 0 { + t.Errorf("entries = %+v, want none: the task was never parked", page.Entries) + } +} + func TestProcessParksWhenManagedWorkerCapabilityIsUnavailable(t *testing.T) { tests := []struct { name string