fix(daemon): park a task whose agent reported no status
runViaAgent logged the agent's reply status and returned, so a task whose agent could not persist its own outcome (empty status, State Store down) stayed running with no park reason while the run read as complete. A status the task cannot reach from running now parks it, quoting the status and naming the condition.
This commit is contained in:
1 parent
89cbbfc871
commit
7b2af3348d
2 files changed
+112
No files matched your search
@@ -0,0 +1,93 @@
|
|||||||
|
package daemon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||||
|
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||||
|
"github.com/samcharles93/archie-core/internal/events"
|
||||||
|
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||||
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// statusStore records the parks a taskrun reply drove.
|
||||||
|
type statusStore struct {
|
||||||
|
storecontract.TaskStore
|
||||||
|
parks []string
|
||||||
|
class taskstate.ParkClass
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *statusStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
|
||||||
|
|
||||||
|
func (s *statusStore) ParkTask(_ context.Context, _ int64, _, reason string, class taskstate.ParkClass, _ taskstate.Blocker) error {
|
||||||
|
s.parks = append(s.parks, reason)
|
||||||
|
s.class = class
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusBus answers every taskrun request with the same reply.
|
||||||
|
type statusBus struct{ reply []byte }
|
||||||
|
|
||||||
|
func (statusBus) PublishUnique(context.Context, string, string, []byte) error { return nil }
|
||||||
|
|
||||||
|
func (statusBus) Fetch(context.Context) (eventbus.Message, error) { return nil, nil }
|
||||||
|
|
||||||
|
func (b statusBus) Request(context.Context, string, []byte) ([]byte, error) { return b.reply, nil }
|
||||||
|
|
||||||
|
// TestRunViaAgentRequiresTerminalStatus pins that a run which recorded no
|
||||||
|
// outcome is parked rather than reported as complete: the agent applies the
|
||||||
|
// task's own transition, so a status the task cannot reach from running means
|
||||||
|
// nothing landed and the row would stay running forever.
|
||||||
|
func TestRunViaAgentRequiresTerminalStatus(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status string
|
||||||
|
wantPark string
|
||||||
|
wantClass taskstate.ParkClass
|
||||||
|
}{
|
||||||
|
{name: "empty status parks", status: "", wantPark: "no status", wantClass: taskstate.ParkTransient},
|
||||||
|
{name: "still running parks", status: taskstate.Running, wantPark: taskstate.Running},
|
||||||
|
{name: "unknown status parks", status: "done-ish", wantPark: "done-ish"},
|
||||||
|
{name: "completed status completes", status: taskstate.Completed},
|
||||||
|
{name: "pr open status completes", status: taskstate.PROpen},
|
||||||
|
{name: "parked status completes", status: taskstate.Parked},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
reply, err := json.Marshal(taskrun.Response{Status: tc.status})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal response: %v", err)
|
||||||
|
}
|
||||||
|
store := &statusStore{}
|
||||||
|
d := &Daemon{
|
||||||
|
Cfg: config.NewHolder(config.Config{}),
|
||||||
|
Store: store,
|
||||||
|
Tasks: statusBus{reply: reply},
|
||||||
|
Log: slog.New(slog.DiscardHandler),
|
||||||
|
}
|
||||||
|
d.runViaAgent(t.Context(), &workflow.Task{ID: 7, Attempt: 1, Owner: "owner", Repo: "repo", Workflow: "implement"},
|
||||||
|
config.Repo{}, config.AgentProfile{}, nil, "")
|
||||||
|
|
||||||
|
if tc.wantPark == "" {
|
||||||
|
if len(store.parks) != 0 {
|
||||||
|
t.Fatalf("valid status %q parked the task: %v", tc.status, store.parks)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(store.parks) != 1 {
|
||||||
|
t.Fatalf("status %q parked %d times, want 1 (task would stay running)", tc.status, len(store.parks))
|
||||||
|
}
|
||||||
|
if !strings.Contains(store.parks[0], tc.wantPark) {
|
||||||
|
t.Fatalf("park reason %q does not name the condition %q", store.parks[0], tc.wantPark)
|
||||||
|
}
|
||||||
|
if tc.wantClass != "" && store.class != tc.wantClass {
|
||||||
|
t.Fatalf("park class = %q, want %q", store.class, tc.wantClass)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1906,6 +1906,16 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
|
|||||||
d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
|
d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// The agent applies the task's own outcome transition, so a status the
|
||||||
|
// task cannot reach from running means nothing landed: empty, unknown, or
|
||||||
|
// still running. Transient, because the cause is environmental -- a State
|
||||||
|
// Store that refused the transition -- and a retry can record one.
|
||||||
|
if !taskstate.CanTransition(taskstate.Running, resp.Status) {
|
||||||
|
reason := noOutcomeReason(resp.Status)
|
||||||
|
d.Log.Error("taskrun run recorded no outcome", "task", task.ID, "status", resp.Status)
|
||||||
|
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// Reached only past every failure return above, so this is always a
|
// Reached only past every failure return above, so this is always a
|
||||||
// fully decoded, error-free response -- a version is never recorded off
|
// fully decoded, error-free response -- a version is never recorded off
|
||||||
@@ -1918,6 +1928,15 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
|
|||||||
d.Log.Info("taskrun complete", "task", task.ID, "status", resp.Status)
|
d.Log.Info("taskrun complete", "task", task.ID, "status", resp.Status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// noOutcomeReason names why a taskrun reply left the task without an outcome,
|
||||||
|
// quoting the status so the log line points at what the agent actually said.
|
||||||
|
func noOutcomeReason(status string) string {
|
||||||
|
if status == "" {
|
||||||
|
return "agent returned no status: the run could not record an outcome"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status)
|
||||||
|
}
|
||||||
|
|
||||||
// pinTaskProfile pins the task's workflow definition and resolves its agent
|
// pinTaskProfile pins the task's workflow definition and resolves its agent
|
||||||
// profile, parking the task on failure.
|
// profile, parking the task on failure.
|
||||||
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
|
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
|
||||||
|
|||||||
Reference in new issue
Block a user