fix(workflow): a budget stop in a parallel branch asks to continue and resumes the branch (#1838)

This commit is contained in:
sam committed 2026-11-09 14:37:22 +11:00
1 parent a3fdbddfc0
commit bcbd09fc08
4 files changed
+129 -35

No files matched your search

+30 -18
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"path/filepath"
"slices"
@@ -76,7 +77,7 @@ func (a AgentStage) Stage() Stage {
}
// The agent call is a child of the stage's recorded step: the store
// says what the stage's runtime did, not just what the stage did.
req.History = tc.takeHistory(a.Name)
req.History = tc.takeHistory(a.historyKey(tc))
tc.transcript = slices.Clone(req.History)
res, runErr := tc.RunAgentChild(ctx, a.Name, a.Model, modelRef, func() (agentrun.Result, error) {
report := tc.reporter(ctx, a.Name)
@@ -274,7 +275,7 @@ func budgetStopOutcome(stage string, res agentrun.Result) Outcome {
// worktree is kept as it stands, the stage's findings and changed files go to
// the next run as task notes, and its conversation is kept for the run to
// continue from. The run loop records where the workflow resumes.
func awaitBudgetApproval(tc *TaskContext, stage string, res agentrun.Result) {
func awaitBudgetApproval(tc *TaskContext, stage, historyKey string, res agentrun.Result) {
note := fmt.Sprintf("Stage %s stopped on its step budget and was approved to continue. The worktree holds its work so far; carry on from it.", stage)
if summary := strings.TrimSpace(res.Summary); summary != "" {
note += " Where it stood: " + summary
@@ -284,7 +285,7 @@ func awaitBudgetApproval(tc *TaskContext, stage string, res agentrun.Result) {
}
tc.Task.Notes += "- " + note + "\n"
tc.Task.RetryMode = string(taskstate.RetryKeepWorktree)
tc.Task.ResumeHistory = encodeHistory(stage, agentrun.BoundHistory(tc.transcript, agentrun.HistoryBytes(0)))
tc.Task.ResumeHistory = mergeHistories(tc.Task.ResumeHistory, stageHistories{historyKey: agentrun.BoundHistory(tc.transcript, agentrun.HistoryBytes(0))})
detail := budgetStopOutcome(stage, res).Detail
tc.Outcome = Outcome{Status: StatusWaitingApproval, Detail: clip(detail+"\n\nApprove to continue with another full step budget, or reject to stop.", 4000)}
}
@@ -297,7 +298,7 @@ func (a AgentStage) deliverResult(tc *TaskContext, res agentrun.Result) error {
// ReviewResult and OnResult stay behind the passed check, because
// a budget stop is not a success.
if a.ApproveBudget && res.StopReason == stopBudgetExhausted {
awaitBudgetApproval(tc, a.Name, res)
awaitBudgetApproval(tc, a.Name, a.historyKey(tc), res)
return nil
}
if budgetStopped(res) {
@@ -406,25 +407,36 @@ func loadSkillBody(tc *TaskContext) string {
return body
}
// stageHistory is the conversation a budget-stopped stage leaves for its
// continuation, as persisted on the task.
type stageHistory struct {
Stage string `json:"stage"`
Entries []agentrun.TranscriptEntry `json:"entries"`
// stageHistories are the conversations budget-stopped agent steps leave for
// their continuation, by step path, as persisted on the task. Parallel
// branches can stop in the same attempt.
type stageHistories map[string][]agentrun.TranscriptEntry
// historyKey identifies this step's conversation: its path when it runs in a
// workflow, which tells apart the same step in two branches.
func (a AgentStage) historyKey(tc *TaskContext) string {
if tc.stepPath != "" {
return tc.stepPath
}
return a.Name
}
func encodeHistory(stage string, entries []agentrun.TranscriptEntry) json.RawMessage {
// mergeHistories adds the conversations in more to the persisted ones.
func mergeHistories(persisted json.RawMessage, more stageHistories) json.RawMessage {
merged := stageHistories{}
if len(persisted) > 0 {
// A value this package wrote; one it cannot read is dropped.
_ = json.Unmarshal(persisted, &merged)
}
maps.Copy(merged, more)
// Transcript entries hold strings and numbers, which always marshal.
encoded, _ := json.Marshal(stageHistory{Stage: stage, Entries: entries})
encoded, _ := json.Marshal(merged)
return encoded
}
// takeHistory hands the stage the conversation its earlier run left, once.
func (tc *TaskContext) takeHistory(stage string) []agentrun.TranscriptEntry {
if tc.resumeHistory.Stage != stage {
return nil
}
entries := tc.resumeHistory.Entries
tc.resumeHistory = stageHistory{}
// takeHistory hands the step the conversation its earlier run left, once.
func (tc *TaskContext) takeHistory(key string) []agentrun.TranscriptEntry {
entries := tc.resumeHistory[key]
delete(tc.resumeHistory, key)
return entries
}
@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
"strings"
"sync"
"testing"
"github.com/samcharles93/archie-core/internal/config"
@@ -13,19 +14,30 @@ import (
"github.com/samcharles93/archie-core/internal/taskstate"
)
// sequenceRunner answers each agent run with the next result, after the
// conversation entries it reports, and keeps the requests it was given.
// sequenceRunner answers each agent run with the next result queued for its
// mission, after the conversation entries it reports, and keeps the requests
// it was given. Parallel branches run concurrently, so the queue is by mission.
type sequenceRunner struct {
results []agentrun.Result
mu sync.Mutex
results map[string][]agentrun.Result
requests []agentrun.Request
}
func (s *sequenceRunner) Run(_ context.Context, _ string, req agentrun.Request, report agentrun.Reporter) (agentrun.Result, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.requests = append(s.requests, req)
var key string
for mission := range s.results {
if strings.Contains(req.Mission, mission) {
key = mission
}
}
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptUser, Text: "the mission"})
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptTool, Tool: "read", Output: fmt.Sprintf("run %d output", len(s.requests))})
res := s.results[0]
s.results = s.results[1:]
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptTool, Tool: "read", Output: fmt.Sprintf("%s run %d output", key, len(s.requests))})
res := s.results[key][0]
res.Stage = req.Stage
s.results[key] = s.results[key][1:]
return res, nil
}
@@ -44,21 +56,26 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
tests := []struct {
name string
approve bool
parallel bool
reason string
wantStatus string
wantKeep bool
}{
{name: "step budget asks", approve: true, reason: "budget_exhausted", wantStatus: StatusWaitingApproval, wantKeep: true},
{name: "step budget in a parallel branch asks", approve: true, parallel: true, reason: "budget_exhausted", wantStatus: StatusWaitingApproval, wantKeep: true},
{name: "not opted in parks", reason: "budget_exhausted", wantStatus: StatusParked},
{name: "wall clock parks", approve: true, reason: "timed_out", wantStatus: StatusParked},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
runner := &sequenceRunner{results: []agentrun.Result{stopped(tt.reason), passed}}
runner := &sequenceRunner{results: map[string][]agentrun.Result{"do it": {stopped(tt.reason), passed}}}
if tt.parallel {
runner.results = map[string][]agentrun.Result{"left": {passed}, "right": {stopped(tt.reason), passed}}
}
tk := &task.Task{ID: 1, Attempt: 1}
newContext := func() *TaskContext {
return &TaskContext{
Task: tk, Store: &fakeStepStore{}, Log: slog.Default(), Agent: runner,
Dir: "/work", Task: tk, Store: &lockedStepStore{}, Log: slog.Default(), Agent: runner,
Cfg: config.Config{Models: map[string]string{"default": "test/model"}},
}
}
@@ -67,6 +84,18 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
{Name: "setup", Run: func(context.Context, *TaskContext) error { setups++; return nil }},
AgentStage{Name: "build", ApproveBudget: tt.approve, Mission: func(*TaskContext) string { return "do it" }}.Stage(),
}}
if tt.parallel {
registry := StepRegistry{AgentRunStepName: newAgentRunStage}
definition, err := ParseDefinition("id: fan\nrepository: none\nsteps:\n - parallel:\n a: [{type: agent.run, settings: {mission: left, read_only: true}}]\n b: [{type: agent.run, settings: {mission: right, read_only: true}}]\n", registry)
if err != nil {
t.Fatal(err)
}
compiled, err := Compile(definition, registry)
if err != nil {
t.Fatal(err)
}
wf.Stages = append(wf.Stages[:1], compiled.Stages...)
}
tc := newContext()
Run(context.Background(), wf, tc)
@@ -80,8 +109,12 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
if !tt.wantKeep {
return
}
if tk.ResumeFrom != "build" {
t.Errorf("resume point = %q, want the stopped step", tk.ResumeFrom)
wantFrom, stoppedMission, wantRuns := "build", "do it", 2
if tt.parallel {
wantFrom, stoppedMission, wantRuns = "parallel", "right", 3
}
if tk.ResumeFrom != wantFrom {
t.Errorf("resume point = %q, want %q", tk.ResumeFrom, wantFrom)
}
if kind := taskstate.ApprovalKind(StatusWaitingApproval, "", tk.RetryMode); kind != taskstate.ApprovalBudget {
t.Errorf("approval kind = %q, want %q", kind, taskstate.ApprovalBudget)
@@ -103,14 +136,23 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
if tk.ResumeFrom != "" || len(tk.ResumeHistory) != 0 {
t.Errorf("resume point %q and history survive the continuation", tk.ResumeFrom)
}
if len(runner.requests) != 2 {
t.Fatalf("agent ran %d times, want 2", len(runner.requests))
if len(runner.requests) != wantRuns {
t.Fatalf("agent ran %d times, want %d: a continuation does not run finished branches again", len(runner.requests), wantRuns)
}
if first := runner.requests[0].History; len(first) != 0 {
var stoppedRuns []agentrun.Request
for _, req := range runner.requests {
if strings.Contains(req.Mission, stoppedMission) {
stoppedRuns = append(stoppedRuns, req)
}
}
if len(stoppedRuns) != 2 {
t.Fatalf("stopped step ran %d times, want 2", len(stoppedRuns))
}
if first := stoppedRuns[0].History; len(first) != 0 {
t.Errorf("first run got history %v", first)
}
history := runner.requests[1].History
if len(history) != 1 || history[0].Output != "run 1 output" {
history := stoppedRuns[1].History
if len(history) != 1 || !strings.HasPrefix(history[0].Output, stoppedMission+" run ") {
t.Errorf("continued run history = %+v, want the stopped run's tool output without its mission", history)
}
})
+26
View File
@@ -2,6 +2,7 @@ package workflow
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
@@ -387,6 +388,10 @@ func compileSwitch(step StepRecord, registry StepRegistry) (Stage, error) {
func runBranch(ctx context.Context, tc *TaskContext, stages []Stage) error {
for _, stage := range stages {
done := joinStepPath(tc.stepPath, branchStepName(tc.workflowBranch, stage.Name))
if tc.resumeDone[done] {
continue
}
parentID := tc.StepID
stepID, _, err := tc.startChildStep(ctx, task.StepKindStage, branchStepName(tc.workflowBranch, stage.Name))
if err != nil {
@@ -408,6 +413,15 @@ func runBranch(ctx context.Context, tc *TaskContext, stages []Stage) error {
if tc.Outcome.Status != "" {
return nil
}
if runErr == nil {
// A continuation after a budget stop skips what finished.
if tc.stepResults == nil {
tc.stepResults = map[string]StepResult{}
}
if _, ok := tc.stepResults[done]; !ok {
tc.stepResults[done] = StepResult{}
}
}
}
return nil
}
@@ -436,6 +450,8 @@ func (tc *TaskContext) branch(name string) *TaskContext {
clone.RunUsage = agentrun.Usage{}
clone.Outcome = Outcome{}
clone.stepResults = maps.Clone(tc.stepResults)
clone.resumeHistory = maps.Clone(tc.resumeHistory)
clone.notesAtFork = tc.Task.Notes
clone.Log = tc.Log.With("branch", name)
return &clone
}
@@ -474,5 +490,15 @@ func (tc *TaskContext) join(branch *TaskContext, before map[string]any) error {
if tc.Outcome.Status == "" {
tc.Outcome = branch.Outcome
}
// A branch that stopped on its step budget leaves what the continuation
// needs, as a stage in the run's own context does.
if branch.Outcome.Status == StatusWaitingApproval && tc.Outcome.Status == StatusWaitingApproval {
tc.Task.RetryMode = branch.Task.RetryMode
tc.Task.Notes += strings.TrimPrefix(branch.Task.Notes, branch.notesAtFork)
var histories stageHistories
if json.Unmarshal(branch.Task.ResumeHistory, &histories) == nil {
tc.Task.ResumeHistory = mergeHistories(tc.Task.ResumeHistory, histories)
}
}
return err
}
+16 -2
View File
@@ -93,7 +93,10 @@ type TaskContext struct {
// transcript is the conversation of the agent run in progress, and
// resumeHistory the one a budget stop left for the stage it continues.
transcript []agentrun.TranscriptEntry
resumeHistory stageHistory
resumeHistory stageHistories
// resumeDone holds the branch steps a budget stop left finished, which
// the continuation does not run again; read-only once the run starts.
resumeDone map[string]bool
// skippedWhen is the when condition that skipped the stage whose body just
// ran; empty when the stage ran. wrapStep sets it and the engine reads it
// to record the step as skipped. It is per-attempt scratch state, reset by
@@ -151,7 +154,10 @@ type TaskContext struct {
// qualified by the branch it runs in and by every step above it. It is a
// workflow.call's durable identity, so a retry of one call site starts
// one child.
stepPath string
stepPath string
// notesAtFork is the task notes a parallel branch started with, so join
// can take what the branch added.
notesAtFork string
workflowBranch string
// BuildSummary is the builder agent's finish summary -- the PR body.
BuildSummary string
@@ -505,6 +511,14 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
return nil, fmt.Errorf("resume history: %w", err)
}
}
for key := range tc.stepResults {
if strings.Contains(key, "/") {
if tc.resumeDone == nil {
tc.resumeDone = map[string]bool{}
}
tc.resumeDone[key] = true
}
}
tc.Log.Info("resuming", "from", from, "skipped", at)
return stages[at:], nil
}