feat(workflow): agent-call children, call steps, and the waited-on callee sweep

Plan item 4 of docs/prds/execution-tree-state-machine.md (archie-core-1216),
plus the callee half of CancelExecution that plan item 5 deferred until the
tree could decide it.

Every agent call a stage makes records itself: AgentStage's bridge, the
baseline-fix auto-fix run, the adversarial reviewer and a workflow.call each
start a child StepExecution of kind agent or call under the stage's own step
-- TaskContext.StepID, which the engine sets before the stage body starts and
clears when it returns -- and finish it with the call's own outcome and the
tokens it accounted. A child's recording write failure parks the execution
the same way a stage's does: an agent call that ran unrecorded is the
invisibility the step table exists to end. A stage invoked outside the
engine's recorded run -- the stage unit tests -- records no child and runs
unchanged.

The call step is what makes the callee half of the PRD's Cancellation section
decidable, and this lands it: a call step closes when its callee ends, so an
open call step is a callee the caller waits on (wait:true) and a closed one
is a wait:false callee that runs on. CancelExecution reads the open call
steps before its sweep closes them and cancels each named callee through its
own CancelExecution in the caller's write's aftermath -- the callee's own
steps, its own events, its own move to the same destination, and the callee's
own wait:true callees beyond it, the depth the enqueue enforces bounding the
recursion. A callee that cannot be cancelled is reported and the caller's own
record stands.

StartStep refuses a call step whose callee is not this run's own (the
callee's call_parent_task_id must name the recording execution): a
compromised container could otherwise record a step naming a foreign
execution and cancel it. StartStepRequest gains called_execution_id (field 6;
non-breaking, the column and its named consumer already exist) and
StepStart carries it.

PRD decisions recorded here:
- The reviewer records as one agent child named "review": the Reviewer
  contract runs one subagent per review, the PRD model's review:1/review:2
  are illustrative of depth-1 children. A completed report closes it
  succeeded, a not-run review failed with the skip reason; the contract
  carries no usage, so tokens are 0.
- A failed-callee call step closes as failed with the callee's detail when
  the wait ends otherwise; the single open case is the caller itself being
  cancelled, whose CancelExecution already swept this step -- a late write
  must not pretend the wait is still the caller's business.
- A call step's callee is cancelled to the same destination the caller's
  action named (parked stays parked, closed_wont_do ends the whole chain),
  and a callee that cannot be cancelled is reported while the caller's own
  record stands either way.
- The agent child's detail is the result summary on success and the error's
  text on failure, with the call's own TokensUsed -- the PRD's only usage
  field, untracked per call for the reviewer contract.
This commit is contained in:
sam committed 2026-09-27 07:14:15 +10:00
1 parent ed12d4de78
commit b8cb7145a6
16 files changed
+566 -91

No files matched your search

+17 -6
View File
@@ -3391,10 +3391,13 @@ type StartStepRequest struct {
// StepExecution, from which the store derives depth.
ParentId int64 `protobuf:"varint,3,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"`
// kind is stage, agent or call.
Kind string `protobuf:"bytes,4,opt,name=kind,proto3" json:"kind,omitempty"`
Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
Kind string `protobuf:"bytes,4,opt,name=kind,proto3" json:"kind,omitempty"`
Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"`
// called_execution_id names the WorkflowExecution a `call` step started;
// the store verifies it is this execution's own callee.
CalledExecutionId int64 `protobuf:"varint,6,opt,name=called_execution_id,json=calledExecutionId,proto3" json:"called_execution_id,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *StartStepRequest) Reset() {
@@ -3462,6 +3465,13 @@ func (x *StartStepRequest) GetName() string {
return ""
}
func (x *StartStepRequest) GetCalledExecutionId() int64 {
if x != nil {
return x.CalledExecutionId
}
return 0
}
type StartStepResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
StepId int64 `protobuf:"varint,1,opt,name=step_id,json=stepId,proto3" json:"step_id,omitempty"`
@@ -11407,13 +11417,14 @@ const file_state_v1_state_proto_rawDesc = "" +
"\x04from\x18\x02 \x01(\tR\x04from\x12\x0e\n" +
"\x02to\x18\x03 \x01(\tR\x02to\x12\x16\n" +
"\x06detail\x18\x04 \x01(\tR\x06detail\"\x14\n" +
"\x12TransitionResponse\"\x94\x01\n" +
"\x12TransitionResponse\"\xc4\x01\n" +
"\x10StartStepRequest\x12!\n" +
"\fexecution_id\x18\x01 \x01(\x03R\vexecutionId\x12\x18\n" +
"\aattempt\x18\x02 \x01(\x03R\aattempt\x12\x1b\n" +
"\tparent_id\x18\x03 \x01(\x03R\bparentId\x12\x12\n" +
"\x04kind\x18\x04 \x01(\tR\x04kind\x12\x12\n" +
"\x04name\x18\x05 \x01(\tR\x04name\"S\n" +
"\x04name\x18\x05 \x01(\tR\x04name\x12.\n" +
"\x13called_execution_id\x18\x06 \x01(\x03R\x11calledExecutionId\"S\n" +
"\x11StartStepResponse\x12\x17\n" +
"\astep_id\x18\x01 \x01(\x03R\x06stepId\x12%\n" +
"\x05event\x18\x02 \x01(\v2\x0f.state.v1.EventR\x05event\"\xac\x01\n" +
+5 -1
View File
@@ -71,7 +71,11 @@ func (a AgentStage) Stage() Stage {
return err
}
req := a.buildRequest(tc, modelRef)
res, runErr := tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(a.Name))
// The agent call is a child of the stage's recorded step: the store
// says what the stage's runtime did, not just what the stage did.
res, runErr := tc.RunAgentChild(ctx, a.Name, func() (agentexec.Result, error) {
return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(a.Name))
})
return a.handleResult(ctx, tc, req, res, runErr, modelRef)
}}
}
+3 -2
View File
@@ -27,6 +27,7 @@ type recordingStore struct {
finishes []StepFinish
insertEr error
stepEr error
finishEr error
stepNames map[int64]string
stepAttempts map[int64]int
}
@@ -67,8 +68,8 @@ func (s *recordingStore) StartStep(_ context.Context, start StepStart) (int64, e
}
func (s *recordingStore) FinishStep(_ context.Context, finish StepFinish) (events.Event, error) {
if s.stepEr != nil {
return events.Event{}, s.stepEr
if s.finishEr != nil || s.stepEr != nil {
return events.Event{}, s.finishEr
}
s.finishes = append(s.finishes, finish)
id := int64(len(s.finishes))
+86 -79
View File
@@ -35,87 +35,94 @@ const (
// fix them via TDD before proceeding. Only parks if the builder cannot fix
// the failures.
func StageBaselineGate() Stage {
return Stage{Name: "baseline", Run: func(ctx context.Context, tc *TaskContext) error {
for _, argv := range tc.Repo.Gate {
if len(argv) == 0 {
continue
}
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
cmd.Dir = tc.Dir
out, err := cmd.CombinedOutput()
if err == nil {
continue
}
tc.Log.Warn("baseline red -- auto-fixing pre-existing gate failure",
"cmd", strings.Join(argv, " "),
"err", clipTail(string(out), baselineWarnLogBytes))
return Stage{Name: "baseline", Run: stageBaselineGateRun}
}
// Run builder to fix the failure via TDD.
mission := fmt.Sprintf(
"The baseline gate check `%s` failed on the repository %s at the base commit "+
"(before any feature work). This is a pre-existing issue -- not caused by your work. "+
"Fix it using TDD:\n\n"+
"1. Write a test that proves the failure exists\n"+
"2. Run the test -- it should FAIL (confirming the bug)\n"+
"3. Fix the root cause\n"+
"4. Run the test -- it should PASS\n"+
"5. Repeat until the gate `%s` passes for all packages\n\n"+
"Gate output:\n%s\n\n"+
"When the gate passes, call finish with status \"passed\".",
strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "),
clip(extractFailingGateOutput(string(out)), baselineMissionBytes),
)
modelRef := tc.Cfg.Models["builder"]
req := agentexec.Request{
Version: agentexec.ProtocolVersion,
TaskID: tc.Task.ID,
Attempt: tc.Task.Attempt,
Stage: "baseline-fix",
Workflow: tc.Task.Workflow,
Model: modelRef,
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
Mission: mission,
Budget: agentexec.Budget{
MaxSteps: tc.Cfg.Budgets.MaxSteps,
WallClock: tc.Cfg.Budgets.WallClock.Std(),
},
Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets),
Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)},
}
res, agentErr := tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix"))
if agentErr != nil && res.Version == 0 {
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
}
tc.Task.TokensUsed += res.TokensUsed
tc.Task.Iterations += res.Iterations
accumulateUsage(&tc.RunUsage, res.Usage)
if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil {
return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr)
}
if agentErr != nil {
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
}
if res.Status != agentexec.StatusPassed {
return fmt.Errorf("baseline red -- %s fails and builder could not auto-fix (status: %s)\n\ngate output:\n%s",
strings.Join(argv, " "), res.Status, clipTail(string(out), baselineParkOutputBytes))
}
// Commit the baseline fix.
changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir,
fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " ")))
if commitErr != nil {
tc.Log.Warn("baseline fix commit failed", "err", commitErr)
}
if changed {
tc.BuildSummary = res.Summary
tc.BaselineFixed = true
tc.captureChanges(ctx, capturedAfterBaselineFix)
}
// stageBaselineGateRun is StageBaselineGate's body, extracted to keep the
// composition's own length off the gate; the auto-fix agent it may dispatch
// records itself under the stage's step.
func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error {
for _, argv := range tc.Repo.Gate {
if len(argv) == 0 {
continue
}
return nil
}}
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
cmd.Dir = tc.Dir
out, err := cmd.CombinedOutput()
if err == nil {
continue
}
tc.Log.Warn("baseline red -- auto-fixing pre-existing gate failure",
"cmd", strings.Join(argv, " "),
"err", clipTail(string(out), baselineWarnLogBytes))
// Run builder to fix the failure via TDD.
mission := fmt.Sprintf(
"The baseline gate check `%s` failed on the repository %s at the base commit "+
"(before any feature work). This is a pre-existing issue -- not caused by your work. "+
"Fix it using TDD:\n\n"+
"1. Write a test that proves the failure exists\n"+
"2. Run the test -- it should FAIL (confirming the bug)\n"+
"3. Fix the root cause\n"+
"4. Run the test -- it should PASS\n"+
"5. Repeat until the gate `%s` passes for all packages\n\n"+
"Gate output:\n%s\n\n"+
"When the gate passes, call finish with status \"passed\".",
strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "),
clip(extractFailingGateOutput(string(out)), baselineMissionBytes),
)
modelRef := tc.Cfg.Models["builder"]
req := agentexec.Request{
Version: agentexec.ProtocolVersion,
TaskID: tc.Task.ID,
Attempt: tc.Task.Attempt,
Stage: "baseline-fix",
Workflow: tc.Task.Workflow,
Model: modelRef,
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
Mission: mission,
Budget: agentexec.Budget{
MaxSteps: tc.Cfg.Budgets.MaxSteps,
WallClock: tc.Cfg.Budgets.WallClock.Std(),
},
Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets),
Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)},
}
res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) {
return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix"))
})
if agentErr != nil && res.Version == 0 {
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
}
tc.Task.TokensUsed += res.TokensUsed
tc.Task.Iterations += res.Iterations
accumulateUsage(&tc.RunUsage, res.Usage)
if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil {
return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr)
}
if agentErr != nil {
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
}
if res.Status != agentexec.StatusPassed {
return fmt.Errorf("baseline red -- %s fails and builder could not auto-fix (status: %s)\n\ngate output:\n%s",
strings.Join(argv, " "), res.Status, clipTail(string(out), baselineParkOutputBytes))
}
// Commit the baseline fix.
changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir,
fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " ")))
if commitErr != nil {
tc.Log.Warn("baseline fix commit failed", "err", commitErr)
}
if changed {
tc.BuildSummary = res.Summary
tc.BaselineFixed = true
tc.captureChanges(ctx, capturedAfterBaselineFix)
}
}
return nil
}
// Implement is the default workflow: plan read-only, post the plan to
+22 -1
View File
@@ -6,6 +6,9 @@ import (
"os"
"path/filepath"
"strings"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// Reviewer runs an adversarial review of a code snapshot in its own
@@ -102,7 +105,25 @@ func runReview(ctx context.Context, tc *TaskContext) (ReviewReport, error) {
IssueText: taskPromptBlock(tc.Task),
MaxSteps: tc.Cfg.Budgets.MaxSteps,
}
return tc.Reviewer.Review(ctx, req), nil
// The reviewer is an agent call like any other: it records itself as a
// child of the review stage's step, so an intervention during review can
// say what the reviewer did. The reviewer runs on a report contract, not
// the agent runner's result, so the recording maps the report's own
// verdict: a review that ran is succeeded, one that never ran is failed,
// and the tokens it accounted are not carried by the contract.
stepID, _, err := tc.startChildStep(ctx, task.StepKindAgent, "review")
if err != nil {
return ReviewReport{}, err
}
report := tc.Reviewer.Review(ctx, req)
to, detail := taskstate.StepSucceeded, report.Summary
if report.Status == ReviewStatusNotRun {
to, detail = taskstate.StepFailed, report.SkipReason
}
if ferr := tc.finishChildStep(ctx, stepID, to, detail, 0); ferr != nil {
return ReviewReport{}, ferr
}
return report, nil
}
// StagePostReviewComments posts the review's line-anchored findings as inline
+6
View File
@@ -38,6 +38,12 @@ type StepStart struct {
ParentID int64
Kind string
Name string
// CalledExecutionID names the WorkflowExecution a `call` step started.
// The store refuses a value that is not the caller's own callee (the
// callee's call_parent_task_id must name this execution), so a step's
// tree cannot cross runs: a cancel of that execution would otherwise
// cancel work the caller never started. Non-call kinds leave it 0.
CalledExecutionID int64
}
// StepFinish moves one step from From to its outcome. ExecutionID names the
+70
View File
@@ -109,6 +109,11 @@ type TaskContext struct {
// Dir/Branch are set by the prepare step.
Dir string
Branch string
// StepID is the StepExecution the engine recorded for the stage whose
// body is running: every agent call the stage makes records itself as
// this step's child. Zero outside a stage body, which is how the stage
// unit tests run their bodies.
StepID int64
// BuildSummary is the builder agent's finish summary -- the PR body.
BuildSummary string
// BuildNoChanges is set when the builder returned StatusPassed but
@@ -318,6 +323,11 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
return
}
publishEvent(tc, startEvent)
// The stage's own step is the parent of every agent or call step the
// stage body records; zero it when the body returns, so a caller that
// outlives the stage cannot parent to a step that has finished.
tc.StepID = stepID
defer func() { tc.StepID = 0 }()
// The stage is bound onto the task logger for exactly the stage's own
// execution and removed afterwards, so every line a stage's code writes
// is selectable by stage (internal/logging.Query.Stage). It is restored
@@ -505,3 +515,63 @@ func extractFailingGateOutput(s string) string {
// goTestOKLine matches a `go test` passing-package summary line, e.g.
// "ok \tgithub.com/example/pkg\t0.004s".
var goTestOKLine = regexp.MustCompile(`^ok\s+\S+`)
// RunAgentChild records one agent call as a child StepExecution of the stage
// this run is executing (docs/prds/execution-tree-state-machine.md, "Model"):
// kind agent, parented to the stage's own step, which the store derives depth
// from, and finished with the call's own outcome and the tokens it
// accounted. A child's recording write failure is the same park a stage's
// is: an agent call that ran unrecorded is the invisibility the step table
// exists to end. A stage invoked outside the engine's recorded run -- which
// is how the stage unit tests run -- records no child and runs its work
// unchanged.
func (tc *TaskContext) RunAgentChild(ctx context.Context, name string, run func() (agentexec.Result, error)) (agentexec.Result, error) {
stepID, _, err := tc.startChildStep(ctx, task.StepKindAgent, name)
if err != nil {
return agentexec.Result{}, err
}
res, runErr := run()
to, detail := taskstate.StepSucceeded, res.Summary
if runErr != nil {
to, detail = taskstate.StepFailed, runErr.Error()
}
if ferr := tc.finishChildStep(ctx, stepID, to, detail, int64(res.TokensUsed)); ferr != nil {
return res, ferr
}
return res, runErr
}
// startChildStep records one agent or call step under the stage step this
// run is executing. A run with no recorded stage step records nothing and
// reports step id 0 without an error, which is how the stage unit tests keep
// running their bodies exactly as they did before the record existed.
func (tc *TaskContext) startChildStep(ctx context.Context, kind, name string) (int64, events.Event, error) {
if tc.Store == nil || tc.StepID == 0 {
return 0, events.Event{}, nil
}
stepID, event, err := tc.Store.StartStep(ctx, task.StepStart{
ExecutionID: tc.Task.ID, Attempt: tc.Task.Attempt, ParentID: tc.StepID, Kind: kind, Name: name,
})
if err != nil {
return 0, events.Event{}, err
}
publishEvent(tc, event)
return stepID, event, nil
}
// finishChildStep closes a child step and publishes the transition's
// persisted event, the same post-commit convention the stage steps use.
func (tc *TaskContext) finishChildStep(ctx context.Context, stepID int64, to taskstate.StepStatus, detail string, tokensUsed int64) error {
if stepID == 0 {
return nil
}
event, err := tc.Store.FinishStep(ctx, task.StepFinish{
StepID: stepID, ExecutionID: tc.Task.ID,
From: taskstate.StepRunning, To: to, Detail: detail, TokensUsed: tokensUsed,
})
if err != nil {
return err
}
publishEvent(tc, event)
return nil
}
+52 -1
View File
@@ -13,6 +13,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// WorkflowCallStepName is the step type that starts another workflow as its
@@ -101,10 +102,60 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex
fmt.Sprintf("started %q as task %d", s.Workflow, callee.ID), started); err != nil {
tc.Log.Warn("workflow call start not persisted", "err", err)
}
// The call is a call StepExecution in the caller's tree, recording the
// callee's execution ID -- the store verifies the callee is this run's
// own callee, which is what makes "cancel the callees the caller waits
// on" decidable: an open call step is a waited-on callee, a closed one
// is a wait:false callee that runs on.
callStep, _, err := tc.startCallStep(ctx, callee.ID)
if err != nil {
return fmt.Errorf("%s: could not be recorded: %w", WorkflowCallStepName, err)
}
if !s.Wait {
// The call step closes at once: the callee runs on, and nothing the
// caller waits on remains open.
if err := tc.finishChildStep(ctx, callStep, taskstate.StepSucceeded,
fmt.Sprintf("started %q (task %d); no wait", s.Workflow, callee.ID), 0); err != nil {
return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, err)
}
return nil
}
return awaitCallee(ctx, s, tc, callee.ID)
// The wait ends when the callee ends, however it ended: a successful
// callee closes the call step as succeeded; a callee that ended otherwise
// closes it as failed with the callee's detail. The one open case is the
// caller itself being cancelled -- its CancelExecution already swept this
// step with the other non-terminal ones, so a late write here must not
// pretend the wait is still the caller's business.
waitErr := awaitCallee(ctx, s, tc, callee.ID)
if waitErr != nil {
if ctx.Err() == nil {
if ferr := tc.finishChildStep(ctx, callStep, taskstate.StepFailed, waitErr.Error(), 0); ferr != nil {
return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, ferr)
}
}
return waitErr
}
if err := tc.finishChildStep(ctx, callStep, taskstate.StepSucceeded,
fmt.Sprintf("%q (task %d) finished", s.Workflow, callee.ID), 0); err != nil {
return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, err)
}
return nil
}
// startCallStep records the call step for a callee this run just started.
func (tc *TaskContext) startCallStep(ctx context.Context, calleeTaskID int64) (int64, events.Event, error) {
if tc.Store == nil {
return 0, events.Event{}, nil
}
stepID, event, err := tc.Store.StartStep(ctx, task.StepStart{
ExecutionID: tc.Task.ID, Attempt: tc.Task.Attempt, ParentID: tc.StepID,
Kind: task.StepKindCall, Name: WorkflowCallStepName, CalledExecutionID: calleeTaskID,
})
if err != nil {
return 0, events.Event{}, err
}
publishEvent(tc, event)
return stepID, event, nil
}
// resolveCallInputs turns the call's saved values into the callee's inputs: a
@@ -7,6 +7,7 @@ import (
"slices"
"testing"
"github.com/samcharles93/archie-core/internal/agentexec"
task "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
)
@@ -114,3 +115,80 @@ func TestRunParksWhenAStepWriteFails(t *testing.T) {
}
var errStepWrite = errors.New("recording write failed")
// The agent-call children (docs/prds/execution-tree-state-machine.md, "Model"):
// every agent call a stage makes is its own StepExecution of kind agent,
// parented to the stage's recorded step, with the call's own outcome and the
// tokens it accounted. Stage "implement" carries agent "implement" one level
// down, the way the model draws it.
func TestRunRecordsAgentChildrenUnderTheStageStep(t *testing.T) {
store := &recordingStore{}
wf := Workflow{Name: "recording", Stages: []Stage{{
Name: "implement",
Run: func(ctx context.Context, tc *TaskContext) error {
_, err := tc.RunAgentChild(ctx, "implement", func() (agentexec.Result, error) {
return agentexec.Result{TokensUsed: 120, Summary: "built"}, nil
})
return err
},
}}}
tc := &TaskContext{
Task: &Task{ID: 41, Attempt: 3, Owner: "acme", Repo: "widgets", Status: StatusRunning},
Store: store, Log: slog.New(slog.DiscardHandler),
}
Run(context.Background(), wf, tc)
if len(store.steps) != 2 {
t.Fatalf("steps = %+v, want the stage and its agent child", store.steps)
}
stageStep, child := store.steps[0], store.steps[1]
if stageStep.Kind != StepKindStage || stageStep.Name != "implement" {
t.Errorf("stage step = %+v, want the recorded stage", stageStep)
}
if child.Kind != StepKindAgent || child.Name != "implement" {
t.Errorf("child step = %+v, want the agent call", child)
}
// The parent is the stage's own step: the store derives depth from it.
if child.ParentID == 0 || child.ParentID == stageStep.ExecutionID {
t.Errorf("child parent = %d, want the stage step's id, not the execution's", child.ParentID)
}
if len(store.finishes) != 2 {
t.Fatalf("finishes = %+v, want the stage's and the child's", store.finishes)
}
var childFinish StepFinish
for _, f := range store.finishes {
if f.StepID == child.ParentID+1 {
childFinish = f
}
}
if childFinish.To != "succeeded" || childFinish.TokensUsed != 120 {
t.Errorf("child finish = %+v, want succeeded with the call's own usage", childFinish)
}
}
// TestRunParksWhenAChildStepWriteFails: a child's recording write parks the
// execution the same way a stage's does -- an agent call that ran unrecorded
// is exactly the invisibility the step table exists to end.
func TestRunParksWhenAChildStepWriteFails(t *testing.T) {
store := &recordingStore{finishEr: errStepWrite}
wf := Workflow{Name: "recording", Stages: []Stage{{
Name: "implement",
Run: func(ctx context.Context, tc *TaskContext) error {
_, err := tc.RunAgentChild(ctx, "implement", func() (agentexec.Result, error) {
return agentexec.Result{}, nil
})
return err
},
}}}
tc := &TaskContext{
Task: &Task{ID: 9, Attempt: 1, Owner: "acme", Repo: "widgets", Status: StatusRunning},
Store: store, Log: slog.New(slog.DiscardHandler),
}
Run(context.Background(), wf, tc)
if len(store.transitions) == 0 || store.transitions[0].to != StatusParked {
t.Fatalf("transitions = %+v, want the execution parked after the child's recording failed", store.transitions)
}
}
@@ -272,3 +272,95 @@ func TestRecoverStaleInterruptsRunningSteps(t *testing.T) {
t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0])
}
}
// A wait:true call step is how the store knows the caller is still waiting on
// its callee: it closes only when the callee ends, so a non-terminal call
// step is a callee the caller waits on. Cancelling the caller sweeps each
// such callee through its own CancelExecution -- the callee's own lifecycle
// -- while a finished call step's callee (wait:false, or already ended) runs
// on. Trees never cross runs: a call step naming a task that is not the
// caller's callee cannot even be recorded (StartStep verifies the callee's
// call_parent_task_id).
func TestCancelExecutionCancelsTheCalleesTheCallerWaitsOn(t *testing.T) {
s := storeFor(t)
caller := runningExecution(t, s)
// The callee: started through the call linkage the way a wait:true step
// does (the store derives the callee from the caller's own row), running
// under its own lifecycle, with a running step of its own.
callee, err := s.StartCall(t.Context(), caller.ID, "implement", nil)
if err != nil {
t.Fatalf("EnqueueCallTask: %v", err)
}
if _, err := s.ClaimNext(t.Context()); err != nil {
t.Fatalf("ClaimNext callee: %v", err)
}
// The claim increments the attempt; read the row back the way the
// container's own request carries it.
callee, err = s.TaskByID(t.Context(), callee.ID)
if err != nil || callee == nil {
t.Fatalf("TaskByID(callee): %+v %v", callee, err)
}
if _, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: callee.ID, Attempt: callee.Attempt, Kind: task.StepKindStage, Name: "implement",
}); err != nil {
t.Fatalf("callee StartStep: %v", err)
}
// The caller's wait:true call step names the callee and is still running.
if _, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: caller.ID, Attempt: caller.Attempt, Kind: task.StepKindCall,
Name: "workflow.call", CalledExecutionID: callee.ID,
}); err != nil {
t.Fatalf("call step: %v", err)
}
// A wait:false call step's callee runs on: its call step already ended.
unwaited, err := s.StartCall(t.Context(), caller.ID, "implement", nil)
if err != nil {
t.Fatal(err)
}
if _, err := s.ClaimNext(t.Context()); err != nil {
t.Fatalf("ClaimNext no-wait callee: %v", err)
}
noWaitStep, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: caller.ID, Attempt: caller.Attempt, Kind: task.StepKindCall,
Name: "workflow.call", CalledExecutionID: unwaited.ID,
})
if err != nil {
t.Fatalf("no-wait call step: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: noWaitStep, ExecutionID: caller.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "started; no wait",
}); err != nil {
t.Fatalf("close the no-wait call step: %v", err)
}
if _, err := s.CancelExecution(t.Context(), caller.ID, "stopped by operator", taskstate.Parked); err != nil {
t.Fatalf("CancelExecution: %v", err)
}
waited, err := s.TaskByID(t.Context(), callee.ID)
if err != nil || waited == nil {
t.Fatalf("TaskByID(callee): %+v %v", waited, err)
}
if waited.Status != taskstate.Parked {
t.Fatalf("the waited-on callee = %q, want parked under its own lifecycle", waited.Status)
}
waitedSteps, err := s.TaskEvents(t.Context(), callee.ID)
if err != nil {
t.Fatalf("callee TaskEvents: %v", err)
}
if len(waitedSteps) == 0 {
t.Error("the callee's steps were not cancelled with their events")
}
unwaitedAfter, err := s.TaskByID(t.Context(), unwaited.ID)
if err != nil || unwaitedAfter == nil {
t.Fatalf("TaskByID no-wait callee: %+v %v", unwaitedAfter, err)
}
if unwaitedAfter.Status != taskstate.Running {
t.Errorf("the wait:false callee = %q, want it running on", unwaitedAfter.Status)
}
}
@@ -294,3 +294,60 @@ func (q *Queries) StartStepExecution(ctx context.Context, id int64) (int64, erro
}
return result.RowsAffected(), nil
}
const stepCalleeParent = `-- name: StepCalleeParent :one
SELECT call_parent_task_id FROM tasks WHERE id = $1
`
// The caller linkage a call step's callee must carry: the callee's
// call_parent_task_id names the execution whose tree records the call. A
// call step naming a task that is not this execution's callee is refused,
// because a cancel of that execution would otherwise cancel work the
// caller never started.
func (q *Queries) StepCalleeParent(ctx context.Context, id int64) (int64, error) {
row := q.db.QueryRow(ctx, stepCalleeParent, id)
var call_parent_task_id int64
err := row.Scan(&call_parent_task_id)
return call_parent_task_id, err
}
const waitingCallSteps = `-- name: WaitingCallSteps :many
SELECT id, called_execution_id FROM step_executions
WHERE execution_id = $1 AND attempt = $2
AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0
`
type WaitingCallStepsParams struct {
ExecutionID int64
Attempt int64
}
type WaitingCallStepsRow struct {
ID int64
CalledExecutionID int64
}
// The call steps of the execution's current attempt that are still open and
// name a callee: a call step closes when the callee ends, so an open one is
// a callee the caller waits on (wait:true). The callee sweep of CancelExecution
// walks these, because a callee whose call step already closed -- wait:false,
// or one that ended before the cancel arrived -- runs on.
func (q *Queries) WaitingCallSteps(ctx context.Context, arg WaitingCallStepsParams) ([]WaitingCallStepsRow, error) {
rows, err := q.db.Query(ctx, waitingCallSteps, arg.ExecutionID, arg.Attempt)
if err != nil {
return nil, err
}
defer rows.Close()
var items []WaitingCallStepsRow
for rows.Next() {
var i WaitingCallStepsRow
if err := rows.Scan(&i.ID, &i.CalledExecutionID); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
@@ -16,6 +16,14 @@ INSERT INTO step_executions (org_id, workspace_id, execution_id, attempt, parent
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'pending')
RETURNING id;
-- name: StepCalleeParent :one
-- The caller linkage a call step's callee must carry: the callee's
-- call_parent_task_id names the execution whose tree records the call. A
-- call step naming a task that is not this execution's callee is refused,
-- because a cancel of that execution would otherwise cancel work the
-- caller never started.
SELECT call_parent_task_id FROM tasks WHERE id = $1;
-- name: StartStepExecution :execrows
-- The pending -> running half of StartStep, guarded by the row's own status:
-- the step transition table routes the pair, so a row that is not pending is
@@ -61,3 +69,13 @@ UPDATE step_executions
SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name;
-- name: WaitingCallSteps :many
-- The call steps of the execution's current attempt that are still open and
-- name a callee: a call step closes when the callee ends, so an open one is
-- a callee the caller waits on (wait:true). The callee sweep of CancelExecution
-- walks these, because a callee whose call step already closed -- wait:false,
-- or one that ended before the cancel arrived -- runs on.
SELECT id, called_execution_id FROM step_executions
WHERE execution_id = $1 AND attempt = $2
AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0;
+55 -1
View File
@@ -56,6 +56,9 @@ func (s *Store) StartStep(ctx context.Context, start task.StepStart) (int64, eve
// execution no longer runs is describing a run that is over.
return 0, events.Event{}, storecontract.ErrStaleTransition
}
if err := s.verifyCallStep(ctx, q, start); err != nil {
return 0, events.Event{}, err
}
depth, err := s.startParent(ctx, q, start)
if err != nil {
@@ -72,7 +75,7 @@ func (s *Store) StartStep(ctx context.Context, start task.StepStart) (int64, eve
OrgID: execution.OrgID, WorkspaceID: execution.WorkspaceID,
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stepParentID(start.ParentID), Depth: int32(depth.depth),
Kind: start.Kind, Name: start.Name, CalledExecutionID: 0,
Kind: start.Kind, Name: start.Name, CalledExecutionID: start.CalledExecutionID,
})
if err != nil {
return 0, events.Event{}, err
@@ -99,6 +102,28 @@ func (s *Store) StartStep(ctx context.Context, start task.StepStart) (int64, eve
return stepID, event, nil
}
// verifyCallStep refuses a call step whose callee is not this run's own:
// the callee's call_parent_task_id must name the recording execution, or the
// call step cannot record (a missing callee is the caller's stale view).
// Without this check a cancel of the recording run would sweep work it never
// started.
func (s *Store) verifyCallStep(ctx context.Context, q *postgresdb.Queries, start task.StepStart) error {
if start.CalledExecutionID == 0 {
return nil
}
calleeParent, err := q.StepCalleeParent(ctx, start.CalledExecutionID)
if errors.Is(err, pgx.ErrNoRows) {
return storecontract.ErrStaleTransition
}
if err != nil {
return err
}
if calleeParent != start.ExecutionID {
return storecontract.ErrStaleTransition
}
return nil
}
// stepParent resolves the enclosing StepExecution a child step names: its
// row is locked in the same transaction, the tree must stay inside one
// execution and one attempt, and depth is derived from the parent, never
@@ -258,6 +283,15 @@ func (s *Store) CancelExecution(ctx context.Context, taskID int64, reason, to st
return nil, fmt.Errorf("%w: %s -> %s is not a transition",
storecontract.ErrIllegalTransition, execution.Status, to)
}
// Read the wait:true callees before the sweep below closes them: an open
// call step is a callee the caller waits on, and one whose call step
// already closed -- wait:false, or one that ended first -- runs on.
waiting, err := q.WaitingCallSteps(ctx, postgresdb.WaitingCallStepsParams{
ExecutionID: taskID, Attempt: execution.Attempt,
})
if err != nil {
return nil, err
}
steps, err := q.CancelAttemptSteps(ctx, postgresdb.CancelAttemptStepsParams{
ExecutionID: taskID, Attempt: execution.Attempt,
@@ -295,5 +329,25 @@ func (s *Store) CancelExecution(ctx context.Context, taskID int64, reason, to st
for _, step := range steps {
ids = append(ids, step.ID)
}
if err := s.cancelWaitedCallees(ctx, waiting, reason, to); err != nil {
return ids, err
}
return ids, nil
}
// cancelWaitedCallees cancels each callee the caller was still waiting on
// through its own CancelExecution, under its own lifecycle, in the caller's
// write's aftermath -- the callee's own steps and its own move to the same
// destination. A chain of calls is bounded by the call depth the enqueue
// enforces, so the recursion is finite; a callee that cannot be cancelled is
// reported, and the caller's own record stands either way.
func (s *Store) cancelWaitedCallees(ctx context.Context, waiting []postgresdb.WaitingCallStepsRow, reason, to string) error {
var failures []error
for _, call := range waiting {
if _, err := s.CancelExecution(ctx, call.CalledExecutionID, reason, to); err != nil {
failures = append(failures, fmt.Errorf("cancel callee %d: %w", call.CalledExecutionID, err))
}
}
return errors.Join(failures...)
}
@@ -117,6 +117,7 @@ func (c *Client) StartStep(ctx context.Context, s task.StepStart) (int64, events
r, err := c.client.StartStep(ctx, &pb.StartStepRequest{
ExecutionId: s.ExecutionID, Attempt: int64(s.Attempt),
ParentId: s.ParentID, Kind: s.Kind, Name: s.Name,
CalledExecutionId: s.CalledExecutionID,
})
if err != nil {
return 0, events.Event{}, unmapError(err)
@@ -211,6 +211,7 @@ func (s *server) StartStep(ctx context.Context, r *pb.StartStepRequest) (*pb.Sta
stepID, event, err := s.deps.Steps.StartStep(ctx, task.StepStart{
ExecutionID: r.ExecutionId, Attempt: int(r.Attempt),
ParentID: r.ParentId, Kind: r.Kind, Name: r.Name,
CalledExecutionID: r.CalledExecutionId,
})
if err != nil {
return nil, s.logErr("StartStep", err)
+3
View File
@@ -537,6 +537,9 @@ message StartStepRequest {
// kind is stage, agent or call.
string kind = 4;
string name = 5;
// called_execution_id names the WorkflowExecution a `call` step started;
// the store verifies it is this execution's own callee.
int64 called_execution_id = 6;
}
message StartStepResponse {
int64 step_id = 1;