fix(workflow): a budget stop in a parallel branch asks to continue and resumes the branch (#1838)

This commit is contained in:
sam committed 2026-11-09 14:37:22 +11:00
1 parent a3fdbddfc0
commit bcbd09fc08
4 files changed
+129 -35

No files matched your search

+30 -18
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"maps"
"os" "os"
"path/filepath" "path/filepath"
"slices" "slices"
@@ -76,7 +77,7 @@ func (a AgentStage) Stage() Stage {
} }
// The agent call is a child of the stage's recorded step: the store // The agent call is a child of the stage's recorded step: the store
// says what the stage's runtime did, not just what the stage did. // says what the stage's runtime did, not just what the stage did.
req.History = tc.takeHistory(a.Name) req.History = tc.takeHistory(a.historyKey(tc))
tc.transcript = slices.Clone(req.History) tc.transcript = slices.Clone(req.History)
res, runErr := tc.RunAgentChild(ctx, a.Name, a.Model, modelRef, func() (agentrun.Result, error) { res, runErr := tc.RunAgentChild(ctx, a.Name, a.Model, modelRef, func() (agentrun.Result, error) {
report := tc.reporter(ctx, a.Name) report := tc.reporter(ctx, a.Name)
@@ -274,7 +275,7 @@ func budgetStopOutcome(stage string, res agentrun.Result) Outcome {
// worktree is kept as it stands, the stage's findings and changed files go to // worktree is kept as it stands, the stage's findings and changed files go to
// the next run as task notes, and its conversation is kept for the run to // the next run as task notes, and its conversation is kept for the run to
// continue from. The run loop records where the workflow resumes. // continue from. The run loop records where the workflow resumes.
func awaitBudgetApproval(tc *TaskContext, stage string, res agentrun.Result) { func awaitBudgetApproval(tc *TaskContext, stage, historyKey string, res agentrun.Result) {
note := fmt.Sprintf("Stage %s stopped on its step budget and was approved to continue. The worktree holds its work so far; carry on from it.", stage) note := fmt.Sprintf("Stage %s stopped on its step budget and was approved to continue. The worktree holds its work so far; carry on from it.", stage)
if summary := strings.TrimSpace(res.Summary); summary != "" { if summary := strings.TrimSpace(res.Summary); summary != "" {
note += " Where it stood: " + summary note += " Where it stood: " + summary
@@ -284,7 +285,7 @@ func awaitBudgetApproval(tc *TaskContext, stage string, res agentrun.Result) {
} }
tc.Task.Notes += "- " + note + "\n" tc.Task.Notes += "- " + note + "\n"
tc.Task.RetryMode = string(taskstate.RetryKeepWorktree) tc.Task.RetryMode = string(taskstate.RetryKeepWorktree)
tc.Task.ResumeHistory = encodeHistory(stage, agentrun.BoundHistory(tc.transcript, agentrun.HistoryBytes(0))) tc.Task.ResumeHistory = mergeHistories(tc.Task.ResumeHistory, stageHistories{historyKey: agentrun.BoundHistory(tc.transcript, agentrun.HistoryBytes(0))})
detail := budgetStopOutcome(stage, res).Detail detail := budgetStopOutcome(stage, res).Detail
tc.Outcome = Outcome{Status: StatusWaitingApproval, Detail: clip(detail+"\n\nApprove to continue with another full step budget, or reject to stop.", 4000)} tc.Outcome = Outcome{Status: StatusWaitingApproval, Detail: clip(detail+"\n\nApprove to continue with another full step budget, or reject to stop.", 4000)}
} }
@@ -297,7 +298,7 @@ func (a AgentStage) deliverResult(tc *TaskContext, res agentrun.Result) error {
// ReviewResult and OnResult stay behind the passed check, because // ReviewResult and OnResult stay behind the passed check, because
// a budget stop is not a success. // a budget stop is not a success.
if a.ApproveBudget && res.StopReason == stopBudgetExhausted { if a.ApproveBudget && res.StopReason == stopBudgetExhausted {
awaitBudgetApproval(tc, a.Name, res) awaitBudgetApproval(tc, a.Name, a.historyKey(tc), res)
return nil return nil
} }
if budgetStopped(res) { if budgetStopped(res) {
@@ -406,25 +407,36 @@ func loadSkillBody(tc *TaskContext) string {
return body return body
} }
// stageHistory is the conversation a budget-stopped stage leaves for its // stageHistories are the conversations budget-stopped agent steps leave for
// continuation, as persisted on the task. // their continuation, by step path, as persisted on the task. Parallel
type stageHistory struct { // branches can stop in the same attempt.
Stage string `json:"stage"` type stageHistories map[string][]agentrun.TranscriptEntry
Entries []agentrun.TranscriptEntry `json:"entries"`
// historyKey identifies this step's conversation: its path when it runs in a
// workflow, which tells apart the same step in two branches.
func (a AgentStage) historyKey(tc *TaskContext) string {
if tc.stepPath != "" {
return tc.stepPath
}
return a.Name
} }
func encodeHistory(stage string, entries []agentrun.TranscriptEntry) json.RawMessage { // mergeHistories adds the conversations in more to the persisted ones.
func mergeHistories(persisted json.RawMessage, more stageHistories) json.RawMessage {
merged := stageHistories{}
if len(persisted) > 0 {
// A value this package wrote; one it cannot read is dropped.
_ = json.Unmarshal(persisted, &merged)
}
maps.Copy(merged, more)
// Transcript entries hold strings and numbers, which always marshal. // Transcript entries hold strings and numbers, which always marshal.
encoded, _ := json.Marshal(stageHistory{Stage: stage, Entries: entries}) encoded, _ := json.Marshal(merged)
return encoded return encoded
} }
// takeHistory hands the stage the conversation its earlier run left, once. // takeHistory hands the step the conversation its earlier run left, once.
func (tc *TaskContext) takeHistory(stage string) []agentrun.TranscriptEntry { func (tc *TaskContext) takeHistory(key string) []agentrun.TranscriptEntry {
if tc.resumeHistory.Stage != stage { entries := tc.resumeHistory[key]
return nil delete(tc.resumeHistory, key)
}
entries := tc.resumeHistory.Entries
tc.resumeHistory = stageHistory{}
return entries return entries
} }
@@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"strings" "strings"
"sync"
"testing" "testing"
"github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/config"
@@ -13,19 +14,30 @@ import (
"github.com/samcharles93/archie-core/internal/taskstate" "github.com/samcharles93/archie-core/internal/taskstate"
) )
// sequenceRunner answers each agent run with the next result, after the // sequenceRunner answers each agent run with the next result queued for its
// conversation entries it reports, and keeps the requests it was given. // mission, after the conversation entries it reports, and keeps the requests
// it was given. Parallel branches run concurrently, so the queue is by mission.
type sequenceRunner struct { type sequenceRunner struct {
results []agentrun.Result mu sync.Mutex
results map[string][]agentrun.Result
requests []agentrun.Request requests []agentrun.Request
} }
func (s *sequenceRunner) Run(_ context.Context, _ string, req agentrun.Request, report agentrun.Reporter) (agentrun.Result, error) { func (s *sequenceRunner) Run(_ context.Context, _ string, req agentrun.Request, report agentrun.Reporter) (agentrun.Result, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.requests = append(s.requests, req) s.requests = append(s.requests, req)
var key string
for mission := range s.results {
if strings.Contains(req.Mission, mission) {
key = mission
}
}
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptUser, Text: "the mission"}) report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptUser, Text: "the mission"})
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptTool, Tool: "read", Output: fmt.Sprintf("run %d output", len(s.requests))}) report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptTool, Tool: "read", Output: fmt.Sprintf("%s run %d output", key, len(s.requests))})
res := s.results[0] res := s.results[key][0]
s.results = s.results[1:] res.Stage = req.Stage
s.results[key] = s.results[key][1:]
return res, nil return res, nil
} }
@@ -44,21 +56,26 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
approve bool approve bool
parallel bool
reason string reason string
wantStatus string wantStatus string
wantKeep bool wantKeep bool
}{ }{
{name: "step budget asks", approve: true, reason: "budget_exhausted", wantStatus: StatusWaitingApproval, wantKeep: true}, {name: "step budget asks", approve: true, reason: "budget_exhausted", wantStatus: StatusWaitingApproval, wantKeep: true},
{name: "step budget in a parallel branch asks", approve: true, parallel: true, reason: "budget_exhausted", wantStatus: StatusWaitingApproval, wantKeep: true},
{name: "not opted in parks", reason: "budget_exhausted", wantStatus: StatusParked}, {name: "not opted in parks", reason: "budget_exhausted", wantStatus: StatusParked},
{name: "wall clock parks", approve: true, reason: "timed_out", wantStatus: StatusParked}, {name: "wall clock parks", approve: true, reason: "timed_out", wantStatus: StatusParked},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
runner := &sequenceRunner{results: []agentrun.Result{stopped(tt.reason), passed}} runner := &sequenceRunner{results: map[string][]agentrun.Result{"do it": {stopped(tt.reason), passed}}}
if tt.parallel {
runner.results = map[string][]agentrun.Result{"left": {passed}, "right": {stopped(tt.reason), passed}}
}
tk := &task.Task{ID: 1, Attempt: 1} tk := &task.Task{ID: 1, Attempt: 1}
newContext := func() *TaskContext { newContext := func() *TaskContext {
return &TaskContext{ return &TaskContext{
Task: tk, Store: &fakeStepStore{}, Log: slog.Default(), Agent: runner, Dir: "/work", Task: tk, Store: &lockedStepStore{}, Log: slog.Default(), Agent: runner,
Cfg: config.Config{Models: map[string]string{"default": "test/model"}}, Cfg: config.Config{Models: map[string]string{"default": "test/model"}},
} }
} }
@@ -67,6 +84,18 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
{Name: "setup", Run: func(context.Context, *TaskContext) error { setups++; return nil }}, {Name: "setup", Run: func(context.Context, *TaskContext) error { setups++; return nil }},
AgentStage{Name: "build", ApproveBudget: tt.approve, Mission: func(*TaskContext) string { return "do it" }}.Stage(), AgentStage{Name: "build", ApproveBudget: tt.approve, Mission: func(*TaskContext) string { return "do it" }}.Stage(),
}} }}
if tt.parallel {
registry := StepRegistry{AgentRunStepName: newAgentRunStage}
definition, err := ParseDefinition("id: fan\nrepository: none\nsteps:\n - parallel:\n a: [{type: agent.run, settings: {mission: left, read_only: true}}]\n b: [{type: agent.run, settings: {mission: right, read_only: true}}]\n", registry)
if err != nil {
t.Fatal(err)
}
compiled, err := Compile(definition, registry)
if err != nil {
t.Fatal(err)
}
wf.Stages = append(wf.Stages[:1], compiled.Stages...)
}
tc := newContext() tc := newContext()
Run(context.Background(), wf, tc) Run(context.Background(), wf, tc)
@@ -80,8 +109,12 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
if !tt.wantKeep { if !tt.wantKeep {
return return
} }
if tk.ResumeFrom != "build" { wantFrom, stoppedMission, wantRuns := "build", "do it", 2
t.Errorf("resume point = %q, want the stopped step", tk.ResumeFrom) if tt.parallel {
wantFrom, stoppedMission, wantRuns = "parallel", "right", 3
}
if tk.ResumeFrom != wantFrom {
t.Errorf("resume point = %q, want %q", tk.ResumeFrom, wantFrom)
} }
if kind := taskstate.ApprovalKind(StatusWaitingApproval, "", tk.RetryMode); kind != taskstate.ApprovalBudget { if kind := taskstate.ApprovalKind(StatusWaitingApproval, "", tk.RetryMode); kind != taskstate.ApprovalBudget {
t.Errorf("approval kind = %q, want %q", kind, taskstate.ApprovalBudget) t.Errorf("approval kind = %q, want %q", kind, taskstate.ApprovalBudget)
@@ -103,14 +136,23 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
if tk.ResumeFrom != "" || len(tk.ResumeHistory) != 0 { if tk.ResumeFrom != "" || len(tk.ResumeHistory) != 0 {
t.Errorf("resume point %q and history survive the continuation", tk.ResumeFrom) t.Errorf("resume point %q and history survive the continuation", tk.ResumeFrom)
} }
if len(runner.requests) != 2 { if len(runner.requests) != wantRuns {
t.Fatalf("agent ran %d times, want 2", len(runner.requests)) t.Fatalf("agent ran %d times, want %d: a continuation does not run finished branches again", len(runner.requests), wantRuns)
} }
if first := runner.requests[0].History; len(first) != 0 { var stoppedRuns []agentrun.Request
for _, req := range runner.requests {
if strings.Contains(req.Mission, stoppedMission) {
stoppedRuns = append(stoppedRuns, req)
}
}
if len(stoppedRuns) != 2 {
t.Fatalf("stopped step ran %d times, want 2", len(stoppedRuns))
}
if first := stoppedRuns[0].History; len(first) != 0 {
t.Errorf("first run got history %v", first) t.Errorf("first run got history %v", first)
} }
history := runner.requests[1].History history := stoppedRuns[1].History
if len(history) != 1 || history[0].Output != "run 1 output" { if len(history) != 1 || !strings.HasPrefix(history[0].Output, stoppedMission+" run ") {
t.Errorf("continued run history = %+v, want the stopped run's tool output without its mission", history) t.Errorf("continued run history = %+v, want the stopped run's tool output without its mission", history)
} }
}) })
+26
View File
@@ -2,6 +2,7 @@ package workflow
import ( import (
"context" "context"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"maps" "maps"
@@ -387,6 +388,10 @@ func compileSwitch(step StepRecord, registry StepRegistry) (Stage, error) {
func runBranch(ctx context.Context, tc *TaskContext, stages []Stage) error { func runBranch(ctx context.Context, tc *TaskContext, stages []Stage) error {
for _, stage := range stages { for _, stage := range stages {
done := joinStepPath(tc.stepPath, branchStepName(tc.workflowBranch, stage.Name))
if tc.resumeDone[done] {
continue
}
parentID := tc.StepID parentID := tc.StepID
stepID, _, err := tc.startChildStep(ctx, task.StepKindStage, branchStepName(tc.workflowBranch, stage.Name)) stepID, _, err := tc.startChildStep(ctx, task.StepKindStage, branchStepName(tc.workflowBranch, stage.Name))
if err != nil { if err != nil {
@@ -408,6 +413,15 @@ func runBranch(ctx context.Context, tc *TaskContext, stages []Stage) error {
if tc.Outcome.Status != "" { if tc.Outcome.Status != "" {
return nil return nil
} }
if runErr == nil {
// A continuation after a budget stop skips what finished.
if tc.stepResults == nil {
tc.stepResults = map[string]StepResult{}
}
if _, ok := tc.stepResults[done]; !ok {
tc.stepResults[done] = StepResult{}
}
}
} }
return nil return nil
} }
@@ -436,6 +450,8 @@ func (tc *TaskContext) branch(name string) *TaskContext {
clone.RunUsage = agentrun.Usage{} clone.RunUsage = agentrun.Usage{}
clone.Outcome = Outcome{} clone.Outcome = Outcome{}
clone.stepResults = maps.Clone(tc.stepResults) clone.stepResults = maps.Clone(tc.stepResults)
clone.resumeHistory = maps.Clone(tc.resumeHistory)
clone.notesAtFork = tc.Task.Notes
clone.Log = tc.Log.With("branch", name) clone.Log = tc.Log.With("branch", name)
return &clone return &clone
} }
@@ -474,5 +490,15 @@ func (tc *TaskContext) join(branch *TaskContext, before map[string]any) error {
if tc.Outcome.Status == "" { if tc.Outcome.Status == "" {
tc.Outcome = branch.Outcome tc.Outcome = branch.Outcome
} }
// A branch that stopped on its step budget leaves what the continuation
// needs, as a stage in the run's own context does.
if branch.Outcome.Status == StatusWaitingApproval && tc.Outcome.Status == StatusWaitingApproval {
tc.Task.RetryMode = branch.Task.RetryMode
tc.Task.Notes += strings.TrimPrefix(branch.Task.Notes, branch.notesAtFork)
var histories stageHistories
if json.Unmarshal(branch.Task.ResumeHistory, &histories) == nil {
tc.Task.ResumeHistory = mergeHistories(tc.Task.ResumeHistory, histories)
}
}
return err return err
} }
+16 -2
View File
@@ -93,7 +93,10 @@ type TaskContext struct {
// transcript is the conversation of the agent run in progress, and // transcript is the conversation of the agent run in progress, and
// resumeHistory the one a budget stop left for the stage it continues. // resumeHistory the one a budget stop left for the stage it continues.
transcript []agentrun.TranscriptEntry transcript []agentrun.TranscriptEntry
resumeHistory stageHistory resumeHistory stageHistories
// resumeDone holds the branch steps a budget stop left finished, which
// the continuation does not run again; read-only once the run starts.
resumeDone map[string]bool
// skippedWhen is the when condition that skipped the stage whose body just // skippedWhen is the when condition that skipped the stage whose body just
// ran; empty when the stage ran. wrapStep sets it and the engine reads it // ran; empty when the stage ran. wrapStep sets it and the engine reads it
// to record the step as skipped. It is per-attempt scratch state, reset by // to record the step as skipped. It is per-attempt scratch state, reset by
@@ -151,7 +154,10 @@ type TaskContext struct {
// qualified by the branch it runs in and by every step above it. It is a // qualified by the branch it runs in and by every step above it. It is a
// workflow.call's durable identity, so a retry of one call site starts // workflow.call's durable identity, so a retry of one call site starts
// one child. // one child.
stepPath string stepPath string
// notesAtFork is the task notes a parallel branch started with, so join
// can take what the branch added.
notesAtFork string
workflowBranch string workflowBranch string
// BuildSummary is the builder agent's finish summary -- the PR body. // BuildSummary is the builder agent's finish summary -- the PR body.
BuildSummary string BuildSummary string
@@ -505,6 +511,14 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
return nil, fmt.Errorf("resume history: %w", err) return nil, fmt.Errorf("resume history: %w", err)
} }
} }
for key := range tc.stepResults {
if strings.Contains(key, "/") {
if tc.resumeDone == nil {
tc.resumeDone = map[string]bool{}
}
tc.resumeDone[key] = true
}
}
tc.Log.Info("resuming", "from", from, "skipped", at) tc.Log.Info("resuming", "from", from, "skipped", at)
return stages[at:], nil return stages[at:], nil
} }