fix(workflow): close a budget-stopped stage with its findings as the outcome
This commit is contained in:
1 parent
26d7c51608
commit
c41606ca0f
3 files changed
+172
-21
No files matched your search
@@ -36,8 +36,10 @@ type AgentStage struct {
|
||||
// CaptureTools adds structured-output tools whose calls are returned
|
||||
// as data rather than receiving callbacks into daemon state.
|
||||
CaptureTools func(*TaskContext) []agentrun.CaptureTool
|
||||
// OnResult consumes a successful (passed) result. Parked and idle
|
||||
// results park the workflow before OnResult is called.
|
||||
// OnResult consumes a successful (passed) result. A stop on the stage's
|
||||
// budget closes the run with the partial findings as its outcome before
|
||||
// OnResult is called; any other parked or idle result parks the
|
||||
// workflow.
|
||||
OnResult func(*TaskContext, agentrun.Result) error
|
||||
// ReviewResult gates agent output before OnResult forwards it to the
|
||||
// delivery channels. The daemon calls this hook to review stage output (issue
|
||||
@@ -211,14 +213,66 @@ func persistAppendedNotes(ctx context.Context, tc *TaskContext, res agentrun.Res
|
||||
return nil
|
||||
}
|
||||
|
||||
// Budget stop reasons, matching the runners' vocabulary: the built-in
|
||||
// loop's step budget and both runners' wall clock report one of these.
|
||||
const (
|
||||
stopBudgetExhausted = "budget_exhausted"
|
||||
stopTimedOut = "timed_out"
|
||||
)
|
||||
|
||||
// budgetStopped reports whether res is a stage that exhausted its budget:
|
||||
// the loop's step budget or a runner's wall clock. Such a stop is an
|
||||
// outcome that carries what the stage found, not a runner error.
|
||||
func budgetStopped(res agentrun.Result) bool {
|
||||
return res.StopReason == stopBudgetExhausted || res.StopReason == stopTimedOut
|
||||
}
|
||||
|
||||
// budgetStopOutcome is the outcome a budget-stopped stage closes the run
|
||||
// with: what stopped it and the partial findings it recorded, in the place
|
||||
// a completed run's result would occupy. The status stays parked, so the
|
||||
// outcome is distinguishable from a gate-passed completion.
|
||||
func budgetStopOutcome(stage string, res agentrun.Result) Outcome {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "stage %s stopped on its budget (%s)", stage, res.StopReason)
|
||||
if detail := strings.TrimSpace(res.Detail); detail != "" {
|
||||
b.WriteString(": " + detail)
|
||||
} else if summary := strings.TrimSpace(res.Summary); summary != "" {
|
||||
b.WriteString(": " + summary)
|
||||
}
|
||||
if len(res.AppendedNotes) > 0 {
|
||||
b.WriteString("\n\nPartial findings:")
|
||||
for _, note := range res.AppendedNotes {
|
||||
b.WriteString("\n- " + note)
|
||||
}
|
||||
}
|
||||
if len(res.Changes) > 0 {
|
||||
b.WriteString("\n\nChanged files:")
|
||||
for _, path := range res.Changes {
|
||||
b.WriteString("\n- " + path)
|
||||
}
|
||||
}
|
||||
// The store caps a park reason at 4000; the stop and the findings come
|
||||
// ahead of the changed files, so clipping loses work last.
|
||||
return Outcome{Status: StatusParked, Detail: clip(b.String(), 4000)}
|
||||
}
|
||||
|
||||
// deliverResult checks the run passed, then runs ReviewResult and OnResult.
|
||||
func (a AgentStage) deliverResult(tc *TaskContext, res agentrun.Result) error {
|
||||
if res.Status != agentrun.StatusPassed {
|
||||
// A budget stop is an outcome, not a failure: the stage closes the
|
||||
// run with what it found, without demanding the gate passed.
|
||||
// ReviewResult and OnResult stay behind the passed check, because
|
||||
// a budget stop is not a success.
|
||||
if budgetStopped(res) {
|
||||
tc.Outcome = budgetStopOutcome(a.Name, res)
|
||||
return nil
|
||||
}
|
||||
detail := res.Detail
|
||||
if detail == "" {
|
||||
detail = res.Summary
|
||||
}
|
||||
// A stage that stops short (step budget) has only its notes to show.
|
||||
// A stage that stops without passing has its notes to show alongside
|
||||
// the reason.
|
||||
for _, note := range res.AppendedNotes {
|
||||
detail += "\n- " + note
|
||||
}
|
||||
|
||||
@@ -1,27 +1,115 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// A stage that stops short keeps what it found: the notes it appended reach
|
||||
// the error that parks the run, not only the task notes.
|
||||
func TestDeliverResultNonPassedCarriesFindings(t *testing.T) {
|
||||
res := agentrun.Result{
|
||||
Status: "parked", StopReason: "budget_exhausted",
|
||||
Detail: "stopped after 60/60 steps, 900 tokens",
|
||||
AppendedNotes: []string{"auth lives in internal/auth", "migration needs a backfill"},
|
||||
// fakeBudgetRunner is an agent runner whose stage exhausted its budget.
|
||||
type fakeBudgetRunner struct{ res agentrun.Result }
|
||||
|
||||
func (f fakeBudgetRunner) Run(context.Context, string, agentrun.Request, agentrun.ToolCallReporter) (agentrun.Result, error) {
|
||||
return f.res, nil
|
||||
}
|
||||
|
||||
// A stage that exhausts its budget closes the run with its partial findings
|
||||
// as the outcome: the notes it appended reach tc.Outcome -- where a
|
||||
// completed run's result would go -- and not only the task's notes. The
|
||||
// outcome stays parked, so a budget stop is distinguishable from a
|
||||
// gate-passed completion, and the stage's step is not recorded as failed.
|
||||
func TestBudgetStoppedStageDeliversFindingsAsOutcome(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
stopReason string
|
||||
}{
|
||||
{name: "step budget", stopReason: "budget_exhausted"},
|
||||
{name: "wall clock", stopReason: "timed_out"},
|
||||
}
|
||||
err := AgentStage{Name: "feasibility"}.deliverResult(&TaskContext{}, res)
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
res := agentrun.Result{
|
||||
Version: agentrun.ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "build",
|
||||
Status: "parked", StopReason: tt.stopReason,
|
||||
Detail: "stopped after 60/60 steps, 900 tokens",
|
||||
Changes: []string{"internal/auth/token.go"},
|
||||
AppendedNotes: []string{"auth lives in internal/auth", "migration needs a backfill"},
|
||||
}
|
||||
store := &fakeStepStore{}
|
||||
tc := &TaskContext{
|
||||
Task: &task.Task{ID: 1, Attempt: 1},
|
||||
Store: store,
|
||||
Log: slog.Default(),
|
||||
Agent: fakeBudgetRunner{res: res},
|
||||
Cfg: config.Config{Models: map[string]string{"default": "test/model"}},
|
||||
}
|
||||
wf := Workflow{Name: "implement", Stages: []Stage{
|
||||
AgentStage{Name: "build", Mission: func(*TaskContext) string { return "do it" }}.Stage(),
|
||||
}}
|
||||
|
||||
Run(context.Background(), wf, tc)
|
||||
|
||||
if tc.Outcome.Status != StatusParked {
|
||||
t.Fatalf("outcome status = %q, want %q (a budget stop is not a success)", tc.Outcome.Status, StatusParked)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"stopped after 60/60 steps",
|
||||
"auth lives in internal/auth",
|
||||
"migration needs a backfill",
|
||||
"internal/auth/token.go",
|
||||
} {
|
||||
if !strings.Contains(tc.Outcome.Detail, want) {
|
||||
t.Errorf("outcome detail %q lacks %q", tc.Outcome.Detail, want)
|
||||
}
|
||||
}
|
||||
// The findings reach the outcome, not only the appended notes.
|
||||
for _, want := range res.AppendedNotes {
|
||||
if !strings.Contains(tc.Task.Notes, want) {
|
||||
t.Errorf("task notes %q lack finding %q", tc.Task.Notes, want)
|
||||
}
|
||||
}
|
||||
// The outcome is delivered as the run's terminal transition,
|
||||
// where a completed run's result would land.
|
||||
if len(store.transitions) != 1 {
|
||||
t.Fatalf("transitions = %d, want 1", len(store.transitions))
|
||||
}
|
||||
got := store.transitions[0]
|
||||
if got.from != StatusRunning || got.to != StatusParked {
|
||||
t.Errorf("transition = %s -> %s, want %s -> %s", got.from, got.to, StatusRunning, StatusParked)
|
||||
}
|
||||
if !strings.Contains(got.detail, "auth lives in internal/auth") {
|
||||
t.Errorf("transition detail %q lacks the findings", got.detail)
|
||||
}
|
||||
// A budget stop is an outcome, not a runner error: no step is
|
||||
// recorded as failed.
|
||||
for _, f := range store.finished {
|
||||
if f.To == taskstate.StepFailed {
|
||||
t.Errorf("step %d recorded failed (%q); a budget stop is an outcome, not a failure", f.StepID, f.Detail)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A stage that stops without passing for a reason other than its budget
|
||||
// still parks through the error path, notes included.
|
||||
func TestDeliverResultNonBudgetStopCarriesFindings(t *testing.T) {
|
||||
res := agentrun.Result{
|
||||
Status: "parked", StopReason: "gate_failed",
|
||||
Detail: "gate go test: FAIL",
|
||||
AppendedNotes: []string{"auth lives in internal/auth"},
|
||||
}
|
||||
err := AgentStage{Name: "build"}.deliverResult(&TaskContext{}, res)
|
||||
if err == nil {
|
||||
t.Fatal("want error for parked run")
|
||||
t.Fatal("want error for a stop that is not a budget stop")
|
||||
}
|
||||
for _, want := range res.AppendedNotes {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("error %q lacks finding %q", err, want)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "auth lives in internal/auth") {
|
||||
t.Errorf("error %q lacks the finding", err)
|
||||
}
|
||||
}
|
||||
@@ -20,15 +20,24 @@ import (
|
||||
|
||||
// fakeStepStore records the step writes a branch makes.
|
||||
type fakeStepStore struct {
|
||||
started []task.StepStart
|
||||
finished []task.StepFinish
|
||||
events []events.Event
|
||||
nextID int64
|
||||
started []task.StepStart
|
||||
finished []task.StepFinish
|
||||
events []events.Event
|
||||
transitions []fakeTransition
|
||||
nextID int64
|
||||
}
|
||||
|
||||
// fakeTransition is one recorded task status transition.
|
||||
type fakeTransition struct {
|
||||
taskID int64
|
||||
from, to string
|
||||
detail string
|
||||
}
|
||||
|
||||
func (f *fakeStepStore) Update(context.Context, *task.Task) error { return nil }
|
||||
|
||||
func (f *fakeStepStore) Transition(context.Context, int64, string, string, string) error {
|
||||
func (f *fakeStepStore) Transition(_ context.Context, taskID int64, from, to, detail string) error {
|
||||
f.transitions = append(f.transitions, fakeTransition{taskID: taskID, from: from, to: to, detail: detail})
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user