fix(daemon): park a task whose agent reported no status
runViaAgent logged the agent's reply status and returned, so a task whose agent could not persist its own outcome (empty status, State Store down) stayed running with no park reason while the run read as complete. A status the task cannot reach from running now parks it, quoting the status and naming the condition.
This commit is contained in:
1 parent
89cbbfc871
commit
7b2af3348d
2 files changed
+112
No files matched your search
@@ -0,0 +1,93 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// statusStore records the parks a taskrun reply drove.
|
||||
type statusStore struct {
|
||||
storecontract.TaskStore
|
||||
parks []string
|
||||
class taskstate.ParkClass
|
||||
}
|
||||
|
||||
func (s *statusStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
|
||||
|
||||
func (s *statusStore) ParkTask(_ context.Context, _ int64, _, reason string, class taskstate.ParkClass, _ taskstate.Blocker) error {
|
||||
s.parks = append(s.parks, reason)
|
||||
s.class = class
|
||||
return nil
|
||||
}
|
||||
|
||||
// statusBus answers every taskrun request with the same reply.
|
||||
type statusBus struct{ reply []byte }
|
||||
|
||||
func (statusBus) PublishUnique(context.Context, string, string, []byte) error { return nil }
|
||||
|
||||
func (statusBus) Fetch(context.Context) (eventbus.Message, error) { return nil, nil }
|
||||
|
||||
func (b statusBus) Request(context.Context, string, []byte) ([]byte, error) { return b.reply, nil }
|
||||
|
||||
// TestRunViaAgentRequiresTerminalStatus pins that a run which recorded no
|
||||
// outcome is parked rather than reported as complete: the agent applies the
|
||||
// task's own transition, so a status the task cannot reach from running means
|
||||
// nothing landed and the row would stay running forever.
|
||||
func TestRunViaAgentRequiresTerminalStatus(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status string
|
||||
wantPark string
|
||||
wantClass taskstate.ParkClass
|
||||
}{
|
||||
{name: "empty status parks", status: "", wantPark: "no status", wantClass: taskstate.ParkTransient},
|
||||
{name: "still running parks", status: taskstate.Running, wantPark: taskstate.Running},
|
||||
{name: "unknown status parks", status: "done-ish", wantPark: "done-ish"},
|
||||
{name: "completed status completes", status: taskstate.Completed},
|
||||
{name: "pr open status completes", status: taskstate.PROpen},
|
||||
{name: "parked status completes", status: taskstate.Parked},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
reply, err := json.Marshal(taskrun.Response{Status: tc.status})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal response: %v", err)
|
||||
}
|
||||
store := &statusStore{}
|
||||
d := &Daemon{
|
||||
Cfg: config.NewHolder(config.Config{}),
|
||||
Store: store,
|
||||
Tasks: statusBus{reply: reply},
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
d.runViaAgent(t.Context(), &workflow.Task{ID: 7, Attempt: 1, Owner: "owner", Repo: "repo", Workflow: "implement"},
|
||||
config.Repo{}, config.AgentProfile{}, nil, "")
|
||||
|
||||
if tc.wantPark == "" {
|
||||
if len(store.parks) != 0 {
|
||||
t.Fatalf("valid status %q parked the task: %v", tc.status, store.parks)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(store.parks) != 1 {
|
||||
t.Fatalf("status %q parked %d times, want 1 (task would stay running)", tc.status, len(store.parks))
|
||||
}
|
||||
if !strings.Contains(store.parks[0], tc.wantPark) {
|
||||
t.Fatalf("park reason %q does not name the condition %q", store.parks[0], tc.wantPark)
|
||||
}
|
||||
if tc.wantClass != "" && store.class != tc.wantClass {
|
||||
t.Fatalf("park class = %q, want %q", store.class, tc.wantClass)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1906,6 +1906,16 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
|
||||
d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
// The agent applies the task's own outcome transition, so a status the
|
||||
// task cannot reach from running means nothing landed: empty, unknown, or
|
||||
// still running. Transient, because the cause is environmental -- a State
|
||||
// Store that refused the transition -- and a retry can record one.
|
||||
if !taskstate.CanTransition(taskstate.Running, resp.Status) {
|
||||
reason := noOutcomeReason(resp.Status)
|
||||
d.Log.Error("taskrun run recorded no outcome", "task", task.ID, "status", resp.Status)
|
||||
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
||||
return
|
||||
}
|
||||
|
||||
// Reached only past every failure return above, so this is always a
|
||||
// fully decoded, error-free response -- a version is never recorded off
|
||||
@@ -1918,6 +1928,15 @@ func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo conf
|
||||
d.Log.Info("taskrun complete", "task", task.ID, "status", resp.Status)
|
||||
}
|
||||
|
||||
// noOutcomeReason names why a taskrun reply left the task without an outcome,
|
||||
// quoting the status so the log line points at what the agent actually said.
|
||||
func noOutcomeReason(status string) string {
|
||||
if status == "" {
|
||||
return "agent returned no status: the run could not record an outcome"
|
||||
}
|
||||
return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status)
|
||||
}
|
||||
|
||||
// pinTaskProfile pins the task's workflow definition and resolves its agent
|
||||
// profile, parking the task on failure.
|
||||
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
|
||||
|
||||
Reference in new issue
Block a user