fix(daemon): record every park reason in the attempt's own log

An operator who asks "why did task 1 attempt 3 park?" and downloads that
attempt's log got a zero-byte file, because nothing daemon-side wrote to a
task log at all: the container's system-log subscription was the only
writer, so an attempt that parked before any container produced output --
a worktree prepare failure, an unavailable worker, a taskrun error -- left
the file the sink had created empty and the reason only in the daemon-wide
log.

Both park paths now write the reason into that attempt's sink. The entry is
written after the guarded transition lands, so a park another actor won
does not leave a log claiming an outcome the task board contradicts -- the
same race the parked event is already withheld for.
This commit is contained in:
sam committed 2026-09-18 01:49:49 +10:00
1 parent 3917afeeb3
commit ff27bd03fe
2 files changed
+114 -2

No files matched your search

+28
View File
@@ -1038,6 +1038,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
d.Log.Warn("managed worker park transition failed", "task", task.ID, "err", err)
return
}
d.recordPark(ctx, task.ID, reason)
d.emit(events.Event{
Kind: events.KindParked, TaskID: task.ID,
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
@@ -1051,6 +1052,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
d.Log.Warn("agent task transport park transition failed", "task", task.ID, "err", err)
return
}
d.recordPark(ctx, task.ID, reason)
d.emit(events.Event{
Kind: events.KindParked, TaskID: task.ID,
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
@@ -1263,7 +1265,33 @@ func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason strin
if !errors.Is(err, storecontract.ErrStaleTransition) {
d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err)
}
return
}
// Recorded only once the transition landed. An entry claiming the task
// parked would be a lie when another actor moved it out of running first
// -- the same race the parked event is withheld for.
d.recordPark(ctx, taskID, reason)
}
// recordPark writes one park reason into the task attempt's own log. Every
// place that parks a task calls it, including the two capability parks in
// process() that write their transition directly instead of going through
// parkRunningTask -- a park an operator cannot read the reason for is the same
// defect wherever it happens. Nothing daemon-side used to write to a task log
// at all: the container's system-log subscription was the only writer, so an
// attempt that parked before any container produced output left an empty log
// file and an unexplained park reason.
//
// A task with no open sink (a retry this instance never dispatched, or logging
// unwired) silently drops the entry, which is TaskRegistry.Write's own
// contract.
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
d.TaskLogs.Write(ctx, taskID, logging.Entry{
Time: time.Now(),
Level: slog.LevelError.String(),
Message: "task parked: " + reason,
Fields: map[string]any{"component": "daemon", "task": taskID},
})
}
// containerEnv returns the environment variables passed to agent containers.
+86 -2
View File
@@ -80,16 +80,100 @@ func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) {
task := &workflow.Task{ID: 5, Attempt: 2}
closeFn := d.openTaskLog(task)
if ok := reg.Write(5, logging.Entry{Message: "x"}); !ok {
if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "x"}); !ok {
t.Fatal("Write() = false right after openTaskLog, want true (sink should be open)")
}
closeFn()
if ok := reg.Write(5, logging.Entry{Message: "y"}); ok {
if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "y"}); ok {
t.Error("Write() = true after openTaskLog's closer ran, want false (sink should be closed)")
}
}
// TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog pins the answer to the
// question the task log exists for: "why did this attempt park?" Every park
// used to be written only to the daemon-wide log, so an attempt that parked
// before its container produced any output -- a worktree prepare failure, an
// unavailable worker, a taskrun error -- left a task log that the sink had
// created and nothing had written to. The dashboard offered that zero-byte
// file as a download, and the reason lived somewhere the operator could not
// hand to anyone.
func TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog(t *testing.T) {
d, st, _ := testDaemon(t, 3, 0)
d.Log = slog.New(slog.DiscardHandler)
d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{})
ctx := t.Context()
if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil {
t.Fatal(err)
}
task, err := st.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("ClaimNext = (%+v, %v)", task, err)
}
// The sink is opened for the attempt by process() and closed when the run
// ends; the park happens in between, which is the only window it is read in.
closeLog := d.openTaskLog(task)
defer closeLog()
const reason = "worktree prepare failed: checkout branch fix/42-x: reference not found"
d.parkRunningTask(ctx, task.ID, reason)
page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{})
if err != nil {
t.Fatalf("TaskLog: %v", err)
}
if !page.Found {
t.Fatalf("attempt %d has no log file, want the reason recorded in one", task.Attempt)
}
if len(page.Entries) != 1 {
t.Fatalf("entries = %+v, want exactly the park reason", page.Entries)
}
entry := page.Entries[0]
if !strings.Contains(entry.Message, reason) {
t.Errorf("entry = %q, want it to carry the park reason %q", entry.Message, reason)
}
if entry.Level != slog.LevelError.String() {
t.Errorf("entry level = %q, want %q", entry.Level, slog.LevelError.String())
}
if entry.Fields["component"] != "daemon" {
t.Errorf("entry component = %v, want the daemon to be identifiable in the attempt's own log", entry.Fields["component"])
}
}
// TestParkDoesNotRecordAParkThatLosesTheRace keeps the new task-log entry as
// truthful as the parked event it accompanies. If another actor moved the task
// out of running first, the guarded transition reports ErStoreStaleTransition
// and nothing was parked -- so "task parked: <reason>" must not appear in the
// attempt's log either, or the log claims an outcome the task board contradicts.
func TestParkDoesNotRecordAParkThatLosesTheRace(t *testing.T) {
d, st, _ := testDaemon(t, 3, 0)
d.Log = slog.New(slog.DiscardHandler)
d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{})
ctx := t.Context()
if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil {
t.Fatal(err)
}
// Left queued, not claimed: the running -> parked transition cannot match.
task, err := st.TaskByIssue(ctx, "acme", "widget", 42)
if err != nil || task == nil {
t.Fatalf("TaskByIssue = (%+v, %v)", task, err)
}
closeLog := d.openTaskLog(task)
defer closeLog()
d.parkRunningTask(ctx, task.ID, "worktree prepare failed: simulated")
page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{})
if err != nil {
t.Fatalf("TaskLog: %v", err)
}
if len(page.Entries) != 0 {
t.Errorf("entries = %+v, want none: the task was never parked", page.Entries)
}
}
func TestProcessParksWhenManagedWorkerCapabilityIsUnavailable(t *testing.T) {
tests := []struct {
name string