fix(daemon): record every park reason in the attempt's own log
An operator who asks "why did task 1 attempt 3 park?" and downloads that attempt's log got a zero-byte file, because nothing daemon-side wrote to a task log at all: the container's system-log subscription was the only writer, so an attempt that parked before any container produced output -- a worktree prepare failure, an unavailable worker, a taskrun error -- left the file the sink had created empty and the reason only in the daemon-wide log. Both park paths now write the reason into that attempt's sink. The entry is written after the guarded transition lands, so a park another actor won does not leave a log claiming an outcome the task board contradicts -- the same race the parked event is already withheld for.
This commit is contained in:
1 parent
3917afeeb3
commit
ff27bd03fe
2 files changed
+114
-2
No files matched your search
@@ -1038,6 +1038,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
|
||||
d.Log.Warn("managed worker park transition failed", "task", task.ID, "err", err)
|
||||
return
|
||||
}
|
||||
d.recordPark(ctx, task.ID, reason)
|
||||
d.emit(events.Event{
|
||||
Kind: events.KindParked, TaskID: task.ID,
|
||||
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
|
||||
@@ -1051,6 +1052,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
|
||||
d.Log.Warn("agent task transport park transition failed", "task", task.ID, "err", err)
|
||||
return
|
||||
}
|
||||
d.recordPark(ctx, task.ID, reason)
|
||||
d.emit(events.Event{
|
||||
Kind: events.KindParked, TaskID: task.ID,
|
||||
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
|
||||
@@ -1263,7 +1265,33 @@ func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason strin
|
||||
if !errors.Is(err, storecontract.ErrStaleTransition) {
|
||||
d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Recorded only once the transition landed. An entry claiming the task
|
||||
// parked would be a lie when another actor moved it out of running first
|
||||
// -- the same race the parked event is withheld for.
|
||||
d.recordPark(ctx, taskID, reason)
|
||||
}
|
||||
|
||||
// recordPark writes one park reason into the task attempt's own log. Every
|
||||
// place that parks a task calls it, including the two capability parks in
|
||||
// process() that write their transition directly instead of going through
|
||||
// parkRunningTask -- a park an operator cannot read the reason for is the same
|
||||
// defect wherever it happens. Nothing daemon-side used to write to a task log
|
||||
// at all: the container's system-log subscription was the only writer, so an
|
||||
// attempt that parked before any container produced output left an empty log
|
||||
// file and an unexplained park reason.
|
||||
//
|
||||
// A task with no open sink (a retry this instance never dispatched, or logging
|
||||
// unwired) silently drops the entry, which is TaskRegistry.Write's own
|
||||
// contract.
|
||||
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
|
||||
d.TaskLogs.Write(ctx, taskID, logging.Entry{
|
||||
Time: time.Now(),
|
||||
Level: slog.LevelError.String(),
|
||||
Message: "task parked: " + reason,
|
||||
Fields: map[string]any{"component": "daemon", "task": taskID},
|
||||
})
|
||||
}
|
||||
|
||||
// containerEnv returns the environment variables passed to agent containers.
|
||||
|
||||
@@ -80,16 +80,100 @@ func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) {
|
||||
task := &workflow.Task{ID: 5, Attempt: 2}
|
||||
|
||||
closeFn := d.openTaskLog(task)
|
||||
if ok := reg.Write(5, logging.Entry{Message: "x"}); !ok {
|
||||
if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "x"}); !ok {
|
||||
t.Fatal("Write() = false right after openTaskLog, want true (sink should be open)")
|
||||
}
|
||||
|
||||
closeFn()
|
||||
if ok := reg.Write(5, logging.Entry{Message: "y"}); ok {
|
||||
if ok := reg.Write(t.Context(), 5, logging.Entry{Message: "y"}); ok {
|
||||
t.Error("Write() = true after openTaskLog's closer ran, want false (sink should be closed)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog pins the answer to the
|
||||
// question the task log exists for: "why did this attempt park?" Every park
|
||||
// used to be written only to the daemon-wide log, so an attempt that parked
|
||||
// before its container produced any output -- a worktree prepare failure, an
|
||||
// unavailable worker, a taskrun error -- left a task log that the sink had
|
||||
// created and nothing had written to. The dashboard offered that zero-byte
|
||||
// file as a download, and the reason lived somewhere the operator could not
|
||||
// hand to anyone.
|
||||
func TestParkRunningTaskRecordsItsReasonInTheTasksOwnLog(t *testing.T) {
|
||||
d, st, _ := testDaemon(t, 3, 0)
|
||||
d.Log = slog.New(slog.DiscardHandler)
|
||||
d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{})
|
||||
|
||||
ctx := t.Context()
|
||||
if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := st.ClaimNext(ctx)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("ClaimNext = (%+v, %v)", task, err)
|
||||
}
|
||||
// The sink is opened for the attempt by process() and closed when the run
|
||||
// ends; the park happens in between, which is the only window it is read in.
|
||||
closeLog := d.openTaskLog(task)
|
||||
defer closeLog()
|
||||
|
||||
const reason = "worktree prepare failed: checkout branch fix/42-x: reference not found"
|
||||
d.parkRunningTask(ctx, task.ID, reason)
|
||||
|
||||
page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{})
|
||||
if err != nil {
|
||||
t.Fatalf("TaskLog: %v", err)
|
||||
}
|
||||
if !page.Found {
|
||||
t.Fatalf("attempt %d has no log file, want the reason recorded in one", task.Attempt)
|
||||
}
|
||||
if len(page.Entries) != 1 {
|
||||
t.Fatalf("entries = %+v, want exactly the park reason", page.Entries)
|
||||
}
|
||||
entry := page.Entries[0]
|
||||
if !strings.Contains(entry.Message, reason) {
|
||||
t.Errorf("entry = %q, want it to carry the park reason %q", entry.Message, reason)
|
||||
}
|
||||
if entry.Level != slog.LevelError.String() {
|
||||
t.Errorf("entry level = %q, want %q", entry.Level, slog.LevelError.String())
|
||||
}
|
||||
if entry.Fields["component"] != "daemon" {
|
||||
t.Errorf("entry component = %v, want the daemon to be identifiable in the attempt's own log", entry.Fields["component"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestParkDoesNotRecordAParkThatLosesTheRace keeps the new task-log entry as
|
||||
// truthful as the parked event it accompanies. If another actor moved the task
|
||||
// out of running first, the guarded transition reports ErStoreStaleTransition
|
||||
// and nothing was parked -- so "task parked: <reason>" must not appear in the
|
||||
// attempt's log either, or the log claims an outcome the task board contradicts.
|
||||
func TestParkDoesNotRecordAParkThatLosesTheRace(t *testing.T) {
|
||||
d, st, _ := testDaemon(t, 3, 0)
|
||||
d.Log = slog.New(slog.DiscardHandler)
|
||||
d.TaskLogs = logging.NewTaskRegistry(t.TempDir(), logging.NewFeed(10), logging.TaskSinkOptions{})
|
||||
|
||||
ctx := t.Context()
|
||||
if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Left queued, not claimed: the running -> parked transition cannot match.
|
||||
task, err := st.TaskByIssue(ctx, "acme", "widget", 42)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("TaskByIssue = (%+v, %v)", task, err)
|
||||
}
|
||||
closeLog := d.openTaskLog(task)
|
||||
defer closeLog()
|
||||
|
||||
d.parkRunningTask(ctx, task.ID, "worktree prepare failed: simulated")
|
||||
|
||||
page, err := d.TaskLogs.TaskLog(ctx, task.ID, task.Attempt, logging.Query{})
|
||||
if err != nil {
|
||||
t.Fatalf("TaskLog: %v", err)
|
||||
}
|
||||
if len(page.Entries) != 0 {
|
||||
t.Errorf("entries = %+v, want none: the task was never parked", page.Entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessParksWhenManagedWorkerCapabilityIsUnavailable(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in new issue
Block a user