fix(workflow): a budget stop in a parallel branch asks to continue and resumes the branch (#1838)
This commit is contained in:
4 files changed
+129
-35
No files matched your search
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -76,7 +77,7 @@ func (a AgentStage) Stage() Stage {
|
||||
}
|
||||
// The agent call is a child of the stage's recorded step: the store
|
||||
// says what the stage's runtime did, not just what the stage did.
|
||||
req.History = tc.takeHistory(a.Name)
|
||||
req.History = tc.takeHistory(a.historyKey(tc))
|
||||
tc.transcript = slices.Clone(req.History)
|
||||
res, runErr := tc.RunAgentChild(ctx, a.Name, a.Model, modelRef, func() (agentrun.Result, error) {
|
||||
report := tc.reporter(ctx, a.Name)
|
||||
@@ -274,7 +275,7 @@ func budgetStopOutcome(stage string, res agentrun.Result) Outcome {
|
||||
// worktree is kept as it stands, the stage's findings and changed files go to
|
||||
// the next run as task notes, and its conversation is kept for the run to
|
||||
// continue from. The run loop records where the workflow resumes.
|
||||
func awaitBudgetApproval(tc *TaskContext, stage string, res agentrun.Result) {
|
||||
func awaitBudgetApproval(tc *TaskContext, stage, historyKey string, res agentrun.Result) {
|
||||
note := fmt.Sprintf("Stage %s stopped on its step budget and was approved to continue. The worktree holds its work so far; carry on from it.", stage)
|
||||
if summary := strings.TrimSpace(res.Summary); summary != "" {
|
||||
note += " Where it stood: " + summary
|
||||
@@ -284,7 +285,7 @@ func awaitBudgetApproval(tc *TaskContext, stage string, res agentrun.Result) {
|
||||
}
|
||||
tc.Task.Notes += "- " + note + "\n"
|
||||
tc.Task.RetryMode = string(taskstate.RetryKeepWorktree)
|
||||
tc.Task.ResumeHistory = encodeHistory(stage, agentrun.BoundHistory(tc.transcript, agentrun.HistoryBytes(0)))
|
||||
tc.Task.ResumeHistory = mergeHistories(tc.Task.ResumeHistory, stageHistories{historyKey: agentrun.BoundHistory(tc.transcript, agentrun.HistoryBytes(0))})
|
||||
detail := budgetStopOutcome(stage, res).Detail
|
||||
tc.Outcome = Outcome{Status: StatusWaitingApproval, Detail: clip(detail+"\n\nApprove to continue with another full step budget, or reject to stop.", 4000)}
|
||||
}
|
||||
@@ -297,7 +298,7 @@ func (a AgentStage) deliverResult(tc *TaskContext, res agentrun.Result) error {
|
||||
// ReviewResult and OnResult stay behind the passed check, because
|
||||
// a budget stop is not a success.
|
||||
if a.ApproveBudget && res.StopReason == stopBudgetExhausted {
|
||||
awaitBudgetApproval(tc, a.Name, res)
|
||||
awaitBudgetApproval(tc, a.Name, a.historyKey(tc), res)
|
||||
return nil
|
||||
}
|
||||
if budgetStopped(res) {
|
||||
@@ -406,25 +407,36 @@ func loadSkillBody(tc *TaskContext) string {
|
||||
return body
|
||||
}
|
||||
|
||||
// stageHistory is the conversation a budget-stopped stage leaves for its
|
||||
// continuation, as persisted on the task.
|
||||
type stageHistory struct {
|
||||
Stage string `json:"stage"`
|
||||
Entries []agentrun.TranscriptEntry `json:"entries"`
|
||||
// stageHistories are the conversations budget-stopped agent steps leave for
|
||||
// their continuation, by step path, as persisted on the task. Parallel
|
||||
// branches can stop in the same attempt.
|
||||
type stageHistories map[string][]agentrun.TranscriptEntry
|
||||
|
||||
// historyKey identifies this step's conversation: its path when it runs in a
|
||||
// workflow, which tells apart the same step in two branches.
|
||||
func (a AgentStage) historyKey(tc *TaskContext) string {
|
||||
if tc.stepPath != "" {
|
||||
return tc.stepPath
|
||||
}
|
||||
return a.Name
|
||||
}
|
||||
|
||||
func encodeHistory(stage string, entries []agentrun.TranscriptEntry) json.RawMessage {
|
||||
// mergeHistories adds the conversations in more to the persisted ones.
|
||||
func mergeHistories(persisted json.RawMessage, more stageHistories) json.RawMessage {
|
||||
merged := stageHistories{}
|
||||
if len(persisted) > 0 {
|
||||
// A value this package wrote; one it cannot read is dropped.
|
||||
_ = json.Unmarshal(persisted, &merged)
|
||||
}
|
||||
maps.Copy(merged, more)
|
||||
// Transcript entries hold strings and numbers, which always marshal.
|
||||
encoded, _ := json.Marshal(stageHistory{Stage: stage, Entries: entries})
|
||||
encoded, _ := json.Marshal(merged)
|
||||
return encoded
|
||||
}
|
||||
|
||||
// takeHistory hands the stage the conversation its earlier run left, once.
|
||||
func (tc *TaskContext) takeHistory(stage string) []agentrun.TranscriptEntry {
|
||||
if tc.resumeHistory.Stage != stage {
|
||||
return nil
|
||||
}
|
||||
entries := tc.resumeHistory.Entries
|
||||
tc.resumeHistory = stageHistory{}
|
||||
// takeHistory hands the step the conversation its earlier run left, once.
|
||||
func (tc *TaskContext) takeHistory(key string) []agentrun.TranscriptEntry {
|
||||
entries := tc.resumeHistory[key]
|
||||
delete(tc.resumeHistory, key)
|
||||
return entries
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
@@ -13,19 +14,30 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// sequenceRunner answers each agent run with the next result, after the
|
||||
// conversation entries it reports, and keeps the requests it was given.
|
||||
// sequenceRunner answers each agent run with the next result queued for its
|
||||
// mission, after the conversation entries it reports, and keeps the requests
|
||||
// it was given. Parallel branches run concurrently, so the queue is by mission.
|
||||
type sequenceRunner struct {
|
||||
results []agentrun.Result
|
||||
mu sync.Mutex
|
||||
results map[string][]agentrun.Result
|
||||
requests []agentrun.Request
|
||||
}
|
||||
|
||||
func (s *sequenceRunner) Run(_ context.Context, _ string, req agentrun.Request, report agentrun.Reporter) (agentrun.Result, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.requests = append(s.requests, req)
|
||||
var key string
|
||||
for mission := range s.results {
|
||||
if strings.Contains(req.Mission, mission) {
|
||||
key = mission
|
||||
}
|
||||
}
|
||||
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptUser, Text: "the mission"})
|
||||
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptTool, Tool: "read", Output: fmt.Sprintf("run %d output", len(s.requests))})
|
||||
res := s.results[0]
|
||||
s.results = s.results[1:]
|
||||
report.ReportTranscript(agentrun.TranscriptEntry{Kind: agentrun.TranscriptTool, Tool: "read", Output: fmt.Sprintf("%s run %d output", key, len(s.requests))})
|
||||
res := s.results[key][0]
|
||||
res.Stage = req.Stage
|
||||
s.results[key] = s.results[key][1:]
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -44,21 +56,26 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
approve bool
|
||||
parallel bool
|
||||
reason string
|
||||
wantStatus string
|
||||
wantKeep bool
|
||||
}{
|
||||
{name: "step budget asks", approve: true, reason: "budget_exhausted", wantStatus: StatusWaitingApproval, wantKeep: true},
|
||||
{name: "step budget in a parallel branch asks", approve: true, parallel: true, reason: "budget_exhausted", wantStatus: StatusWaitingApproval, wantKeep: true},
|
||||
{name: "not opted in parks", reason: "budget_exhausted", wantStatus: StatusParked},
|
||||
{name: "wall clock parks", approve: true, reason: "timed_out", wantStatus: StatusParked},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
runner := &sequenceRunner{results: []agentrun.Result{stopped(tt.reason), passed}}
|
||||
runner := &sequenceRunner{results: map[string][]agentrun.Result{"do it": {stopped(tt.reason), passed}}}
|
||||
if tt.parallel {
|
||||
runner.results = map[string][]agentrun.Result{"left": {passed}, "right": {stopped(tt.reason), passed}}
|
||||
}
|
||||
tk := &task.Task{ID: 1, Attempt: 1}
|
||||
newContext := func() *TaskContext {
|
||||
return &TaskContext{
|
||||
Task: tk, Store: &fakeStepStore{}, Log: slog.Default(), Agent: runner,
|
||||
Dir: "/work", Task: tk, Store: &lockedStepStore{}, Log: slog.Default(), Agent: runner,
|
||||
Cfg: config.Config{Models: map[string]string{"default": "test/model"}},
|
||||
}
|
||||
}
|
||||
@@ -67,6 +84,18 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
|
||||
{Name: "setup", Run: func(context.Context, *TaskContext) error { setups++; return nil }},
|
||||
AgentStage{Name: "build", ApproveBudget: tt.approve, Mission: func(*TaskContext) string { return "do it" }}.Stage(),
|
||||
}}
|
||||
if tt.parallel {
|
||||
registry := StepRegistry{AgentRunStepName: newAgentRunStage}
|
||||
definition, err := ParseDefinition("id: fan\nrepository: none\nsteps:\n - parallel:\n a: [{type: agent.run, settings: {mission: left, read_only: true}}]\n b: [{type: agent.run, settings: {mission: right, read_only: true}}]\n", registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compiled, err := Compile(definition, registry)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wf.Stages = append(wf.Stages[:1], compiled.Stages...)
|
||||
}
|
||||
|
||||
tc := newContext()
|
||||
Run(context.Background(), wf, tc)
|
||||
@@ -80,8 +109,12 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
|
||||
if !tt.wantKeep {
|
||||
return
|
||||
}
|
||||
if tk.ResumeFrom != "build" {
|
||||
t.Errorf("resume point = %q, want the stopped step", tk.ResumeFrom)
|
||||
wantFrom, stoppedMission, wantRuns := "build", "do it", 2
|
||||
if tt.parallel {
|
||||
wantFrom, stoppedMission, wantRuns = "parallel", "right", 3
|
||||
}
|
||||
if tk.ResumeFrom != wantFrom {
|
||||
t.Errorf("resume point = %q, want %q", tk.ResumeFrom, wantFrom)
|
||||
}
|
||||
if kind := taskstate.ApprovalKind(StatusWaitingApproval, "", tk.RetryMode); kind != taskstate.ApprovalBudget {
|
||||
t.Errorf("approval kind = %q, want %q", kind, taskstate.ApprovalBudget)
|
||||
@@ -103,14 +136,23 @@ func TestBudgetStopAsksToContinue(t *testing.T) {
|
||||
if tk.ResumeFrom != "" || len(tk.ResumeHistory) != 0 {
|
||||
t.Errorf("resume point %q and history survive the continuation", tk.ResumeFrom)
|
||||
}
|
||||
if len(runner.requests) != 2 {
|
||||
t.Fatalf("agent ran %d times, want 2", len(runner.requests))
|
||||
if len(runner.requests) != wantRuns {
|
||||
t.Fatalf("agent ran %d times, want %d: a continuation does not run finished branches again", len(runner.requests), wantRuns)
|
||||
}
|
||||
if first := runner.requests[0].History; len(first) != 0 {
|
||||
var stoppedRuns []agentrun.Request
|
||||
for _, req := range runner.requests {
|
||||
if strings.Contains(req.Mission, stoppedMission) {
|
||||
stoppedRuns = append(stoppedRuns, req)
|
||||
}
|
||||
}
|
||||
if len(stoppedRuns) != 2 {
|
||||
t.Fatalf("stopped step ran %d times, want 2", len(stoppedRuns))
|
||||
}
|
||||
if first := stoppedRuns[0].History; len(first) != 0 {
|
||||
t.Errorf("first run got history %v", first)
|
||||
}
|
||||
history := runner.requests[1].History
|
||||
if len(history) != 1 || history[0].Output != "run 1 output" {
|
||||
history := stoppedRuns[1].History
|
||||
if len(history) != 1 || !strings.HasPrefix(history[0].Output, stoppedMission+" run ") {
|
||||
t.Errorf("continued run history = %+v, want the stopped run's tool output without its mission", history)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2,6 +2,7 @@ package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
@@ -387,6 +388,10 @@ func compileSwitch(step StepRecord, registry StepRegistry) (Stage, error) {
|
||||
|
||||
func runBranch(ctx context.Context, tc *TaskContext, stages []Stage) error {
|
||||
for _, stage := range stages {
|
||||
done := joinStepPath(tc.stepPath, branchStepName(tc.workflowBranch, stage.Name))
|
||||
if tc.resumeDone[done] {
|
||||
continue
|
||||
}
|
||||
parentID := tc.StepID
|
||||
stepID, _, err := tc.startChildStep(ctx, task.StepKindStage, branchStepName(tc.workflowBranch, stage.Name))
|
||||
if err != nil {
|
||||
@@ -408,6 +413,15 @@ func runBranch(ctx context.Context, tc *TaskContext, stages []Stage) error {
|
||||
if tc.Outcome.Status != "" {
|
||||
return nil
|
||||
}
|
||||
if runErr == nil {
|
||||
// A continuation after a budget stop skips what finished.
|
||||
if tc.stepResults == nil {
|
||||
tc.stepResults = map[string]StepResult{}
|
||||
}
|
||||
if _, ok := tc.stepResults[done]; !ok {
|
||||
tc.stepResults[done] = StepResult{}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -436,6 +450,8 @@ func (tc *TaskContext) branch(name string) *TaskContext {
|
||||
clone.RunUsage = agentrun.Usage{}
|
||||
clone.Outcome = Outcome{}
|
||||
clone.stepResults = maps.Clone(tc.stepResults)
|
||||
clone.resumeHistory = maps.Clone(tc.resumeHistory)
|
||||
clone.notesAtFork = tc.Task.Notes
|
||||
clone.Log = tc.Log.With("branch", name)
|
||||
return &clone
|
||||
}
|
||||
@@ -474,5 +490,15 @@ func (tc *TaskContext) join(branch *TaskContext, before map[string]any) error {
|
||||
if tc.Outcome.Status == "" {
|
||||
tc.Outcome = branch.Outcome
|
||||
}
|
||||
// A branch that stopped on its step budget leaves what the continuation
|
||||
// needs, as a stage in the run's own context does.
|
||||
if branch.Outcome.Status == StatusWaitingApproval && tc.Outcome.Status == StatusWaitingApproval {
|
||||
tc.Task.RetryMode = branch.Task.RetryMode
|
||||
tc.Task.Notes += strings.TrimPrefix(branch.Task.Notes, branch.notesAtFork)
|
||||
var histories stageHistories
|
||||
if json.Unmarshal(branch.Task.ResumeHistory, &histories) == nil {
|
||||
tc.Task.ResumeHistory = mergeHistories(tc.Task.ResumeHistory, histories)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -93,7 +93,10 @@ type TaskContext struct {
|
||||
// transcript is the conversation of the agent run in progress, and
|
||||
// resumeHistory the one a budget stop left for the stage it continues.
|
||||
transcript []agentrun.TranscriptEntry
|
||||
resumeHistory stageHistory
|
||||
resumeHistory stageHistories
|
||||
// resumeDone holds the branch steps a budget stop left finished, which
|
||||
// the continuation does not run again; read-only once the run starts.
|
||||
resumeDone map[string]bool
|
||||
// skippedWhen is the when condition that skipped the stage whose body just
|
||||
// ran; empty when the stage ran. wrapStep sets it and the engine reads it
|
||||
// to record the step as skipped. It is per-attempt scratch state, reset by
|
||||
@@ -151,7 +154,10 @@ type TaskContext struct {
|
||||
// qualified by the branch it runs in and by every step above it. It is a
|
||||
// workflow.call's durable identity, so a retry of one call site starts
|
||||
// one child.
|
||||
stepPath string
|
||||
stepPath string
|
||||
// notesAtFork is the task notes a parallel branch started with, so join
|
||||
// can take what the branch added.
|
||||
notesAtFork string
|
||||
workflowBranch string
|
||||
// BuildSummary is the builder agent's finish summary -- the PR body.
|
||||
BuildSummary string
|
||||
@@ -505,6 +511,14 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
|
||||
return nil, fmt.Errorf("resume history: %w", err)
|
||||
}
|
||||
}
|
||||
for key := range tc.stepResults {
|
||||
if strings.Contains(key, "/") {
|
||||
if tc.resumeDone == nil {
|
||||
tc.resumeDone = map[string]bool{}
|
||||
}
|
||||
tc.resumeDone[key] = true
|
||||
}
|
||||
}
|
||||
tc.Log.Info("resuming", "from", from, "skipped", at)
|
||||
return stages[at:], nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user