diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index 1dbd94afb..f0a259265 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -3391,10 +3391,13 @@ type StartStepRequest struct { // StepExecution, from which the store derives depth. ParentId int64 `protobuf:"varint,3,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"` // kind is stage, agent or call. - Kind string `protobuf:"bytes,4,opt,name=kind,proto3" json:"kind,omitempty"` - Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Kind string `protobuf:"bytes,4,opt,name=kind,proto3" json:"kind,omitempty"` + Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` + // called_execution_id names the WorkflowExecution a `call` step started; + // the store verifies it is this execution's own callee. + CalledExecutionId int64 `protobuf:"varint,6,opt,name=called_execution_id,json=calledExecutionId,proto3" json:"called_execution_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *StartStepRequest) Reset() { @@ -3462,6 +3465,13 @@ func (x *StartStepRequest) GetName() string { return "" } +func (x *StartStepRequest) GetCalledExecutionId() int64 { + if x != nil { + return x.CalledExecutionId + } + return 0 +} + type StartStepResponse struct { state protoimpl.MessageState `protogen:"open.v1"` StepId int64 `protobuf:"varint,1,opt,name=step_id,json=stepId,proto3" json:"step_id,omitempty"` @@ -11407,13 +11417,14 @@ const file_state_v1_state_proto_rawDesc = "" + "\x04from\x18\x02 \x01(\tR\x04from\x12\x0e\n" + "\x02to\x18\x03 \x01(\tR\x02to\x12\x16\n" + "\x06detail\x18\x04 \x01(\tR\x06detail\"\x14\n" + - "\x12TransitionResponse\"\x94\x01\n" + + "\x12TransitionResponse\"\xc4\x01\n" + "\x10StartStepRequest\x12!\n" + "\fexecution_id\x18\x01 \x01(\x03R\vexecutionId\x12\x18\n" + "\aattempt\x18\x02 \x01(\x03R\aattempt\x12\x1b\n" + "\tparent_id\x18\x03 \x01(\x03R\bparentId\x12\x12\n" + "\x04kind\x18\x04 \x01(\tR\x04kind\x12\x12\n" + - "\x04name\x18\x05 \x01(\tR\x04name\"S\n" + + "\x04name\x18\x05 \x01(\tR\x04name\x12.\n" + + "\x13called_execution_id\x18\x06 \x01(\x03R\x11calledExecutionId\"S\n" + "\x11StartStepResponse\x12\x17\n" + "\astep_id\x18\x01 \x01(\x03R\x06stepId\x12%\n" + "\x05event\x18\x02 \x01(\v2\x0f.state.v1.EventR\x05event\"\xac\x01\n" + diff --git a/internal/domain/workflow/agent.go b/internal/domain/workflow/agent.go index 11385432c..797332d91 100644 --- a/internal/domain/workflow/agent.go +++ b/internal/domain/workflow/agent.go @@ -71,7 +71,11 @@ func (a AgentStage) Stage() Stage { return err } req := a.buildRequest(tc, modelRef) - res, runErr := tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(a.Name)) + // The agent call is a child of the stage's recorded step: the store + // says what the stage's runtime did, not just what the stage did. + res, runErr := tc.RunAgentChild(ctx, a.Name, func() (agentexec.Result, error) { + return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter(a.Name)) + }) return a.handleResult(ctx, tc, req, res, runErr, modelRef) }} } diff --git a/internal/domain/workflow/changes_test.go b/internal/domain/workflow/changes_test.go index 29bcd62b1..84451946b 100644 --- a/internal/domain/workflow/changes_test.go +++ b/internal/domain/workflow/changes_test.go @@ -27,6 +27,7 @@ type recordingStore struct { finishes []StepFinish insertEr error stepEr error + finishEr error stepNames map[int64]string stepAttempts map[int64]int } @@ -67,8 +68,8 @@ func (s *recordingStore) StartStep(_ context.Context, start StepStart) (int64, e } func (s *recordingStore) FinishStep(_ context.Context, finish StepFinish) (events.Event, error) { - if s.stepEr != nil { - return events.Event{}, s.stepEr + if s.finishEr != nil || s.stepEr != nil { + return events.Event{}, s.finishEr } s.finishes = append(s.finishes, finish) id := int64(len(s.finishes)) diff --git a/internal/domain/workflow/implement.go b/internal/domain/workflow/implement.go index c221c98e7..65efe9d87 100644 --- a/internal/domain/workflow/implement.go +++ b/internal/domain/workflow/implement.go @@ -35,87 +35,94 @@ const ( // fix them via TDD before proceeding. Only parks if the builder cannot fix // the failures. func StageBaselineGate() Stage { - return Stage{Name: "baseline", Run: func(ctx context.Context, tc *TaskContext) error { - for _, argv := range tc.Repo.Gate { - if len(argv) == 0 { - continue - } - cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) - cmd.Dir = tc.Dir - out, err := cmd.CombinedOutput() - if err == nil { - continue - } - tc.Log.Warn("baseline red -- auto-fixing pre-existing gate failure", - "cmd", strings.Join(argv, " "), - "err", clipTail(string(out), baselineWarnLogBytes)) + return Stage{Name: "baseline", Run: stageBaselineGateRun} +} - // Run builder to fix the failure via TDD. - mission := fmt.Sprintf( - "The baseline gate check `%s` failed on the repository %s at the base commit "+ - "(before any feature work). This is a pre-existing issue -- not caused by your work. "+ - "Fix it using TDD:\n\n"+ - "1. Write a test that proves the failure exists\n"+ - "2. Run the test -- it should FAIL (confirming the bug)\n"+ - "3. Fix the root cause\n"+ - "4. Run the test -- it should PASS\n"+ - "5. Repeat until the gate `%s` passes for all packages\n\n"+ - "Gate output:\n%s\n\n"+ - "When the gate passes, call finish with status \"passed\".", - strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "), - clip(extractFailingGateOutput(string(out)), baselineMissionBytes), - ) - - modelRef := tc.Cfg.Models["builder"] - req := agentexec.Request{ - Version: agentexec.ProtocolVersion, - TaskID: tc.Task.ID, - Attempt: tc.Task.Attempt, - Stage: "baseline-fix", - Workflow: tc.Task.Workflow, - Model: modelRef, - ContextWindow: modelContextBudget(tc.Cfg, modelRef), - Mission: mission, - Budget: agentexec.Budget{ - MaxSteps: tc.Cfg.Budgets.MaxSteps, - WallClock: tc.Cfg.Budgets.WallClock.Std(), - }, - Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets), - Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)}, - } - - res, agentErr := tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix")) - if agentErr != nil && res.Version == 0 { - return fmt.Errorf("baseline-fix agent run: %w", agentErr) - } - tc.Task.TokensUsed += res.TokensUsed - tc.Task.Iterations += res.Iterations - accumulateUsage(&tc.RunUsage, res.Usage) - if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil { - return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr) - } - if agentErr != nil { - return fmt.Errorf("baseline-fix agent run: %w", agentErr) - } - if res.Status != agentexec.StatusPassed { - return fmt.Errorf("baseline red -- %s fails and builder could not auto-fix (status: %s)\n\ngate output:\n%s", - strings.Join(argv, " "), res.Status, clipTail(string(out), baselineParkOutputBytes)) - } - - // Commit the baseline fix. - changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir, - fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " "))) - if commitErr != nil { - tc.Log.Warn("baseline fix commit failed", "err", commitErr) - } - if changed { - tc.BuildSummary = res.Summary - tc.BaselineFixed = true - tc.captureChanges(ctx, capturedAfterBaselineFix) - } +// stageBaselineGateRun is StageBaselineGate's body, extracted to keep the +// composition's own length off the gate; the auto-fix agent it may dispatch +// records itself under the stage's step. +func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error { + for _, argv := range tc.Repo.Gate { + if len(argv) == 0 { + continue } - return nil - }} + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Dir = tc.Dir + out, err := cmd.CombinedOutput() + if err == nil { + continue + } + tc.Log.Warn("baseline red -- auto-fixing pre-existing gate failure", + "cmd", strings.Join(argv, " "), + "err", clipTail(string(out), baselineWarnLogBytes)) + + // Run builder to fix the failure via TDD. + mission := fmt.Sprintf( + "The baseline gate check `%s` failed on the repository %s at the base commit "+ + "(before any feature work). This is a pre-existing issue -- not caused by your work. "+ + "Fix it using TDD:\n\n"+ + "1. Write a test that proves the failure exists\n"+ + "2. Run the test -- it should FAIL (confirming the bug)\n"+ + "3. Fix the root cause\n"+ + "4. Run the test -- it should PASS\n"+ + "5. Repeat until the gate `%s` passes for all packages\n\n"+ + "Gate output:\n%s\n\n"+ + "When the gate passes, call finish with status \"passed\".", + strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "), + clip(extractFailingGateOutput(string(out)), baselineMissionBytes), + ) + + modelRef := tc.Cfg.Models["builder"] + req := agentexec.Request{ + Version: agentexec.ProtocolVersion, + TaskID: tc.Task.ID, + Attempt: tc.Task.Attempt, + Stage: "baseline-fix", + Workflow: tc.Task.Workflow, + Model: modelRef, + ContextWindow: modelContextBudget(tc.Cfg, modelRef), + Mission: mission, + Budget: agentexec.Budget{ + MaxSteps: tc.Cfg.Budgets.MaxSteps, + WallClock: tc.Cfg.Budgets.WallClock.Std(), + }, + Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets), + Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)}, + } + + res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) { + return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix")) + }) + if agentErr != nil && res.Version == 0 { + return fmt.Errorf("baseline-fix agent run: %w", agentErr) + } + tc.Task.TokensUsed += res.TokensUsed + tc.Task.Iterations += res.Iterations + accumulateUsage(&tc.RunUsage, res.Usage) + if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil { + return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr) + } + if agentErr != nil { + return fmt.Errorf("baseline-fix agent run: %w", agentErr) + } + if res.Status != agentexec.StatusPassed { + return fmt.Errorf("baseline red -- %s fails and builder could not auto-fix (status: %s)\n\ngate output:\n%s", + strings.Join(argv, " "), res.Status, clipTail(string(out), baselineParkOutputBytes)) + } + + // Commit the baseline fix. + changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir, + fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " "))) + if commitErr != nil { + tc.Log.Warn("baseline fix commit failed", "err", commitErr) + } + if changed { + tc.BuildSummary = res.Summary + tc.BaselineFixed = true + tc.captureChanges(ctx, capturedAfterBaselineFix) + } + } + return nil } // Implement is the default workflow: plan read-only, post the plan to diff --git a/internal/domain/workflow/review.go b/internal/domain/workflow/review.go index ebede0016..d4ef90a62 100644 --- a/internal/domain/workflow/review.go +++ b/internal/domain/workflow/review.go @@ -6,6 +6,9 @@ import ( "os" "path/filepath" "strings" + + "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/taskstate" ) // Reviewer runs an adversarial review of a code snapshot in its own @@ -102,7 +105,25 @@ func runReview(ctx context.Context, tc *TaskContext) (ReviewReport, error) { IssueText: taskPromptBlock(tc.Task), MaxSteps: tc.Cfg.Budgets.MaxSteps, } - return tc.Reviewer.Review(ctx, req), nil + // The reviewer is an agent call like any other: it records itself as a + // child of the review stage's step, so an intervention during review can + // say what the reviewer did. The reviewer runs on a report contract, not + // the agent runner's result, so the recording maps the report's own + // verdict: a review that ran is succeeded, one that never ran is failed, + // and the tokens it accounted are not carried by the contract. + stepID, _, err := tc.startChildStep(ctx, task.StepKindAgent, "review") + if err != nil { + return ReviewReport{}, err + } + report := tc.Reviewer.Review(ctx, req) + to, detail := taskstate.StepSucceeded, report.Summary + if report.Status == ReviewStatusNotRun { + to, detail = taskstate.StepFailed, report.SkipReason + } + if ferr := tc.finishChildStep(ctx, stepID, to, detail, 0); ferr != nil { + return ReviewReport{}, ferr + } + return report, nil } // StagePostReviewComments posts the review's line-anchored findings as inline diff --git a/internal/domain/workflow/task/step.go b/internal/domain/workflow/task/step.go index 79b863c94..1d9d24ff3 100644 --- a/internal/domain/workflow/task/step.go +++ b/internal/domain/workflow/task/step.go @@ -38,6 +38,12 @@ type StepStart struct { ParentID int64 Kind string Name string + // CalledExecutionID names the WorkflowExecution a `call` step started. + // The store refuses a value that is not the caller's own callee (the + // callee's call_parent_task_id must name this execution), so a step's + // tree cannot cross runs: a cancel of that execution would otherwise + // cancel work the caller never started. Non-call kinds leave it 0. + CalledExecutionID int64 } // StepFinish moves one step from From to its outcome. ExecutionID names the diff --git a/internal/domain/workflow/workflow.go b/internal/domain/workflow/workflow.go index 339e1bc36..03267f96d 100644 --- a/internal/domain/workflow/workflow.go +++ b/internal/domain/workflow/workflow.go @@ -109,6 +109,11 @@ type TaskContext struct { // Dir/Branch are set by the prepare step. Dir string Branch string + // StepID is the StepExecution the engine recorded for the stage whose + // body is running: every agent call the stage makes records itself as + // this step's child. Zero outside a stage body, which is how the stage + // unit tests run their bodies. + StepID int64 // BuildSummary is the builder agent's finish summary -- the PR body. BuildSummary string // BuildNoChanges is set when the builder returned StatusPassed but @@ -318,6 +323,11 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { return } publishEvent(tc, startEvent) + // The stage's own step is the parent of every agent or call step the + // stage body records; zero it when the body returns, so a caller that + // outlives the stage cannot parent to a step that has finished. + tc.StepID = stepID + defer func() { tc.StepID = 0 }() // The stage is bound onto the task logger for exactly the stage's own // execution and removed afterwards, so every line a stage's code writes // is selectable by stage (internal/logging.Query.Stage). It is restored @@ -505,3 +515,63 @@ func extractFailingGateOutput(s string) string { // goTestOKLine matches a `go test` passing-package summary line, e.g. // "ok \tgithub.com/example/pkg\t0.004s". var goTestOKLine = regexp.MustCompile(`^ok\s+\S+`) + +// RunAgentChild records one agent call as a child StepExecution of the stage +// this run is executing (docs/prds/execution-tree-state-machine.md, "Model"): +// kind agent, parented to the stage's own step, which the store derives depth +// from, and finished with the call's own outcome and the tokens it +// accounted. A child's recording write failure is the same park a stage's +// is: an agent call that ran unrecorded is the invisibility the step table +// exists to end. A stage invoked outside the engine's recorded run -- which +// is how the stage unit tests run -- records no child and runs its work +// unchanged. +func (tc *TaskContext) RunAgentChild(ctx context.Context, name string, run func() (agentexec.Result, error)) (agentexec.Result, error) { + stepID, _, err := tc.startChildStep(ctx, task.StepKindAgent, name) + if err != nil { + return agentexec.Result{}, err + } + res, runErr := run() + to, detail := taskstate.StepSucceeded, res.Summary + if runErr != nil { + to, detail = taskstate.StepFailed, runErr.Error() + } + if ferr := tc.finishChildStep(ctx, stepID, to, detail, int64(res.TokensUsed)); ferr != nil { + return res, ferr + } + return res, runErr +} + +// startChildStep records one agent or call step under the stage step this +// run is executing. A run with no recorded stage step records nothing and +// reports step id 0 without an error, which is how the stage unit tests keep +// running their bodies exactly as they did before the record existed. +func (tc *TaskContext) startChildStep(ctx context.Context, kind, name string) (int64, events.Event, error) { + if tc.Store == nil || tc.StepID == 0 { + return 0, events.Event{}, nil + } + stepID, event, err := tc.Store.StartStep(ctx, task.StepStart{ + ExecutionID: tc.Task.ID, Attempt: tc.Task.Attempt, ParentID: tc.StepID, Kind: kind, Name: name, + }) + if err != nil { + return 0, events.Event{}, err + } + publishEvent(tc, event) + return stepID, event, nil +} + +// finishChildStep closes a child step and publishes the transition's +// persisted event, the same post-commit convention the stage steps use. +func (tc *TaskContext) finishChildStep(ctx context.Context, stepID int64, to taskstate.StepStatus, detail string, tokensUsed int64) error { + if stepID == 0 { + return nil + } + event, err := tc.Store.FinishStep(ctx, task.StepFinish{ + StepID: stepID, ExecutionID: tc.Task.ID, + From: taskstate.StepRunning, To: to, Detail: detail, TokensUsed: tokensUsed, + }) + if err != nil { + return err + } + publishEvent(tc, event) + return nil +} diff --git a/internal/domain/workflow/workflow_call.go b/internal/domain/workflow/workflow_call.go index 1a96235e3..f26f3c374 100644 --- a/internal/domain/workflow/workflow_call.go +++ b/internal/domain/workflow/workflow_call.go @@ -13,6 +13,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/taskstate" ) // WorkflowCallStepName is the step type that starts another workflow as its @@ -101,10 +102,60 @@ func runWorkflowCall(ctx context.Context, s workflowCallSettings, tc *TaskContex fmt.Sprintf("started %q as task %d", s.Workflow, callee.ID), started); err != nil { tc.Log.Warn("workflow call start not persisted", "err", err) } + // The call is a call StepExecution in the caller's tree, recording the + // callee's execution ID -- the store verifies the callee is this run's + // own callee, which is what makes "cancel the callees the caller waits + // on" decidable: an open call step is a waited-on callee, a closed one + // is a wait:false callee that runs on. + callStep, _, err := tc.startCallStep(ctx, callee.ID) + if err != nil { + return fmt.Errorf("%s: could not be recorded: %w", WorkflowCallStepName, err) + } if !s.Wait { + // The call step closes at once: the callee runs on, and nothing the + // caller waits on remains open. + if err := tc.finishChildStep(ctx, callStep, taskstate.StepSucceeded, + fmt.Sprintf("started %q (task %d); no wait", s.Workflow, callee.ID), 0); err != nil { + return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, err) + } return nil } - return awaitCallee(ctx, s, tc, callee.ID) + // The wait ends when the callee ends, however it ended: a successful + // callee closes the call step as succeeded; a callee that ended otherwise + // closes it as failed with the callee's detail. The one open case is the + // caller itself being cancelled -- its CancelExecution already swept this + // step with the other non-terminal ones, so a late write here must not + // pretend the wait is still the caller's business. + waitErr := awaitCallee(ctx, s, tc, callee.ID) + if waitErr != nil { + if ctx.Err() == nil { + if ferr := tc.finishChildStep(ctx, callStep, taskstate.StepFailed, waitErr.Error(), 0); ferr != nil { + return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, ferr) + } + } + return waitErr + } + if err := tc.finishChildStep(ctx, callStep, taskstate.StepSucceeded, + fmt.Sprintf("%q (task %d) finished", s.Workflow, callee.ID), 0); err != nil { + return fmt.Errorf("%s: could not be closed: %w", WorkflowCallStepName, err) + } + return nil +} + +// startCallStep records the call step for a callee this run just started. +func (tc *TaskContext) startCallStep(ctx context.Context, calleeTaskID int64) (int64, events.Event, error) { + if tc.Store == nil { + return 0, events.Event{}, nil + } + stepID, event, err := tc.Store.StartStep(ctx, task.StepStart{ + ExecutionID: tc.Task.ID, Attempt: tc.Task.Attempt, ParentID: tc.StepID, + Kind: task.StepKindCall, Name: WorkflowCallStepName, CalledExecutionID: calleeTaskID, + }) + if err != nil { + return 0, events.Event{}, err + } + publishEvent(tc, event) + return stepID, event, nil } // resolveCallInputs turns the call's saved values into the callee's inputs: a diff --git a/internal/domain/workflow/workflow_steps_test.go b/internal/domain/workflow/workflow_steps_test.go index d76dc79cc..9f1a66f42 100644 --- a/internal/domain/workflow/workflow_steps_test.go +++ b/internal/domain/workflow/workflow_steps_test.go @@ -7,6 +7,7 @@ import ( "slices" "testing" + "github.com/samcharles93/archie-core/internal/agentexec" task "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/events" ) @@ -114,3 +115,80 @@ func TestRunParksWhenAStepWriteFails(t *testing.T) { } var errStepWrite = errors.New("recording write failed") + +// The agent-call children (docs/prds/execution-tree-state-machine.md, "Model"): +// every agent call a stage makes is its own StepExecution of kind agent, +// parented to the stage's recorded step, with the call's own outcome and the +// tokens it accounted. Stage "implement" carries agent "implement" one level +// down, the way the model draws it. +func TestRunRecordsAgentChildrenUnderTheStageStep(t *testing.T) { + store := &recordingStore{} + wf := Workflow{Name: "recording", Stages: []Stage{{ + Name: "implement", + Run: func(ctx context.Context, tc *TaskContext) error { + _, err := tc.RunAgentChild(ctx, "implement", func() (agentexec.Result, error) { + return agentexec.Result{TokensUsed: 120, Summary: "built"}, nil + }) + return err + }, + }}} + tc := &TaskContext{ + Task: &Task{ID: 41, Attempt: 3, Owner: "acme", Repo: "widgets", Status: StatusRunning}, + Store: store, Log: slog.New(slog.DiscardHandler), + } + + Run(context.Background(), wf, tc) + + if len(store.steps) != 2 { + t.Fatalf("steps = %+v, want the stage and its agent child", store.steps) + } + stageStep, child := store.steps[0], store.steps[1] + if stageStep.Kind != StepKindStage || stageStep.Name != "implement" { + t.Errorf("stage step = %+v, want the recorded stage", stageStep) + } + if child.Kind != StepKindAgent || child.Name != "implement" { + t.Errorf("child step = %+v, want the agent call", child) + } + // The parent is the stage's own step: the store derives depth from it. + if child.ParentID == 0 || child.ParentID == stageStep.ExecutionID { + t.Errorf("child parent = %d, want the stage step's id, not the execution's", child.ParentID) + } + if len(store.finishes) != 2 { + t.Fatalf("finishes = %+v, want the stage's and the child's", store.finishes) + } + var childFinish StepFinish + for _, f := range store.finishes { + if f.StepID == child.ParentID+1 { + childFinish = f + } + } + if childFinish.To != "succeeded" || childFinish.TokensUsed != 120 { + t.Errorf("child finish = %+v, want succeeded with the call's own usage", childFinish) + } +} + +// TestRunParksWhenAChildStepWriteFails: a child's recording write parks the +// execution the same way a stage's does -- an agent call that ran unrecorded +// is exactly the invisibility the step table exists to end. +func TestRunParksWhenAChildStepWriteFails(t *testing.T) { + store := &recordingStore{finishEr: errStepWrite} + wf := Workflow{Name: "recording", Stages: []Stage{{ + Name: "implement", + Run: func(ctx context.Context, tc *TaskContext) error { + _, err := tc.RunAgentChild(ctx, "implement", func() (agentexec.Result, error) { + return agentexec.Result{}, nil + }) + return err + }, + }}} + tc := &TaskContext{ + Task: &Task{ID: 9, Attempt: 1, Owner: "acme", Repo: "widgets", Status: StatusRunning}, + Store: store, Log: slog.New(slog.DiscardHandler), + } + + Run(context.Background(), wf, tc) + + if len(store.transitions) == 0 || store.transitions[0].to != StatusParked { + t.Fatalf("transitions = %+v, want the execution parked after the child's recording failed", store.transitions) + } +} diff --git a/internal/infrastructure/postgres/cancel_test.go b/internal/infrastructure/postgres/cancel_test.go index dfa3947d6..3a1736654 100644 --- a/internal/infrastructure/postgres/cancel_test.go +++ b/internal/infrastructure/postgres/cancel_test.go @@ -272,3 +272,95 @@ func TestRecoverStaleInterruptsRunningSteps(t *testing.T) { t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0]) } } + +// A wait:true call step is how the store knows the caller is still waiting on +// its callee: it closes only when the callee ends, so a non-terminal call +// step is a callee the caller waits on. Cancelling the caller sweeps each +// such callee through its own CancelExecution -- the callee's own lifecycle +// -- while a finished call step's callee (wait:false, or already ended) runs +// on. Trees never cross runs: a call step naming a task that is not the +// caller's callee cannot even be recorded (StartStep verifies the callee's +// call_parent_task_id). +func TestCancelExecutionCancelsTheCalleesTheCallerWaitsOn(t *testing.T) { + s := storeFor(t) + caller := runningExecution(t, s) + + // The callee: started through the call linkage the way a wait:true step + // does (the store derives the callee from the caller's own row), running + // under its own lifecycle, with a running step of its own. + callee, err := s.StartCall(t.Context(), caller.ID, "implement", nil) + if err != nil { + t.Fatalf("EnqueueCallTask: %v", err) + } + if _, err := s.ClaimNext(t.Context()); err != nil { + t.Fatalf("ClaimNext callee: %v", err) + } + // The claim increments the attempt; read the row back the way the + // container's own request carries it. + callee, err = s.TaskByID(t.Context(), callee.ID) + if err != nil || callee == nil { + t.Fatalf("TaskByID(callee): %+v %v", callee, err) + } + if _, _, err := s.StartStep(t.Context(), task.StepStart{ + ExecutionID: callee.ID, Attempt: callee.Attempt, Kind: task.StepKindStage, Name: "implement", + }); err != nil { + t.Fatalf("callee StartStep: %v", err) + } + + // The caller's wait:true call step names the callee and is still running. + if _, _, err := s.StartStep(t.Context(), task.StepStart{ + ExecutionID: caller.ID, Attempt: caller.Attempt, Kind: task.StepKindCall, + Name: "workflow.call", CalledExecutionID: callee.ID, + }); err != nil { + t.Fatalf("call step: %v", err) + } + + // A wait:false call step's callee runs on: its call step already ended. + unwaited, err := s.StartCall(t.Context(), caller.ID, "implement", nil) + if err != nil { + t.Fatal(err) + } + if _, err := s.ClaimNext(t.Context()); err != nil { + t.Fatalf("ClaimNext no-wait callee: %v", err) + } + noWaitStep, _, err := s.StartStep(t.Context(), task.StepStart{ + ExecutionID: caller.ID, Attempt: caller.Attempt, Kind: task.StepKindCall, + Name: "workflow.call", CalledExecutionID: unwaited.ID, + }) + if err != nil { + t.Fatalf("no-wait call step: %v", err) + } + if _, err := s.FinishStep(t.Context(), task.StepFinish{ + StepID: noWaitStep, ExecutionID: caller.ID, + From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "started; no wait", + }); err != nil { + t.Fatalf("close the no-wait call step: %v", err) + } + + if _, err := s.CancelExecution(t.Context(), caller.ID, "stopped by operator", taskstate.Parked); err != nil { + t.Fatalf("CancelExecution: %v", err) + } + + waited, err := s.TaskByID(t.Context(), callee.ID) + if err != nil || waited == nil { + t.Fatalf("TaskByID(callee): %+v %v", waited, err) + } + if waited.Status != taskstate.Parked { + t.Fatalf("the waited-on callee = %q, want parked under its own lifecycle", waited.Status) + } + waitedSteps, err := s.TaskEvents(t.Context(), callee.ID) + if err != nil { + t.Fatalf("callee TaskEvents: %v", err) + } + if len(waitedSteps) == 0 { + t.Error("the callee's steps were not cancelled with their events") + } + + unwaitedAfter, err := s.TaskByID(t.Context(), unwaited.ID) + if err != nil || unwaitedAfter == nil { + t.Fatalf("TaskByID no-wait callee: %+v %v", unwaitedAfter, err) + } + if unwaitedAfter.Status != taskstate.Running { + t.Errorf("the wait:false callee = %q, want it running on", unwaitedAfter.Status) + } +} diff --git a/internal/infrastructure/postgres/postgresdb/steps.sql.go b/internal/infrastructure/postgres/postgresdb/steps.sql.go index 296994dbf..b52703a23 100644 --- a/internal/infrastructure/postgres/postgresdb/steps.sql.go +++ b/internal/infrastructure/postgres/postgresdb/steps.sql.go @@ -294,3 +294,60 @@ func (q *Queries) StartStepExecution(ctx context.Context, id int64) (int64, erro } return result.RowsAffected(), nil } + +const stepCalleeParent = `-- name: StepCalleeParent :one +SELECT call_parent_task_id FROM tasks WHERE id = $1 +` + +// The caller linkage a call step's callee must carry: the callee's +// call_parent_task_id names the execution whose tree records the call. A +// call step naming a task that is not this execution's callee is refused, +// because a cancel of that execution would otherwise cancel work the +// caller never started. +func (q *Queries) StepCalleeParent(ctx context.Context, id int64) (int64, error) { + row := q.db.QueryRow(ctx, stepCalleeParent, id) + var call_parent_task_id int64 + err := row.Scan(&call_parent_task_id) + return call_parent_task_id, err +} + +const waitingCallSteps = `-- name: WaitingCallSteps :many +SELECT id, called_execution_id FROM step_executions +WHERE execution_id = $1 AND attempt = $2 + AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0 +` + +type WaitingCallStepsParams struct { + ExecutionID int64 + Attempt int64 +} + +type WaitingCallStepsRow struct { + ID int64 + CalledExecutionID int64 +} + +// The call steps of the execution's current attempt that are still open and +// name a callee: a call step closes when the callee ends, so an open one is +// a callee the caller waits on (wait:true). The callee sweep of CancelExecution +// walks these, because a callee whose call step already closed -- wait:false, +// or one that ended before the cancel arrived -- runs on. +func (q *Queries) WaitingCallSteps(ctx context.Context, arg WaitingCallStepsParams) ([]WaitingCallStepsRow, error) { + rows, err := q.db.Query(ctx, waitingCallSteps, arg.ExecutionID, arg.Attempt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []WaitingCallStepsRow + for rows.Next() { + var i WaitingCallStepsRow + if err := rows.Scan(&i.ID, &i.CalledExecutionID); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/infrastructure/postgres/queries/steps.sql b/internal/infrastructure/postgres/queries/steps.sql index 0d65260a2..a846cb44a 100644 --- a/internal/infrastructure/postgres/queries/steps.sql +++ b/internal/infrastructure/postgres/queries/steps.sql @@ -16,6 +16,14 @@ INSERT INTO step_executions (org_id, workspace_id, execution_id, attempt, parent VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'pending') RETURNING id; +-- name: StepCalleeParent :one +-- The caller linkage a call step's callee must carry: the callee's +-- call_parent_task_id names the execution whose tree records the call. A +-- call step naming a task that is not this execution's callee is refused, +-- because a cancel of that execution would otherwise cancel work the +-- caller never started. +SELECT call_parent_task_id FROM tasks WHERE id = $1; + -- name: StartStepExecution :execrows -- The pending -> running half of StartStep, guarded by the row's own status: -- the step transition table routes the pair, so a row that is not pending is @@ -61,3 +69,13 @@ UPDATE step_executions SET status = 'interrupted', finished_at = now() WHERE execution_id = $1 AND attempt = $2 AND status = 'running' RETURNING id, name; + +-- name: WaitingCallSteps :many +-- The call steps of the execution's current attempt that are still open and +-- name a callee: a call step closes when the callee ends, so an open one is +-- a callee the caller waits on (wait:true). The callee sweep of CancelExecution +-- walks these, because a callee whose call step already closed -- wait:false, +-- or one that ended before the cancel arrived -- runs on. +SELECT id, called_execution_id FROM step_executions +WHERE execution_id = $1 AND attempt = $2 + AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0; diff --git a/internal/infrastructure/postgres/steps.go b/internal/infrastructure/postgres/steps.go index b80bf7fae..c920aa38f 100644 --- a/internal/infrastructure/postgres/steps.go +++ b/internal/infrastructure/postgres/steps.go @@ -56,6 +56,9 @@ func (s *Store) StartStep(ctx context.Context, start task.StepStart) (int64, eve // execution no longer runs is describing a run that is over. return 0, events.Event{}, storecontract.ErrStaleTransition } + if err := s.verifyCallStep(ctx, q, start); err != nil { + return 0, events.Event{}, err + } depth, err := s.startParent(ctx, q, start) if err != nil { @@ -72,7 +75,7 @@ func (s *Store) StartStep(ctx context.Context, start task.StepStart) (int64, eve OrgID: execution.OrgID, WorkspaceID: execution.WorkspaceID, ExecutionID: execution.ID, Attempt: execution.Attempt, ParentID: stepParentID(start.ParentID), Depth: int32(depth.depth), - Kind: start.Kind, Name: start.Name, CalledExecutionID: 0, + Kind: start.Kind, Name: start.Name, CalledExecutionID: start.CalledExecutionID, }) if err != nil { return 0, events.Event{}, err @@ -99,6 +102,28 @@ func (s *Store) StartStep(ctx context.Context, start task.StepStart) (int64, eve return stepID, event, nil } +// verifyCallStep refuses a call step whose callee is not this run's own: +// the callee's call_parent_task_id must name the recording execution, or the +// call step cannot record (a missing callee is the caller's stale view). +// Without this check a cancel of the recording run would sweep work it never +// started. +func (s *Store) verifyCallStep(ctx context.Context, q *postgresdb.Queries, start task.StepStart) error { + if start.CalledExecutionID == 0 { + return nil + } + calleeParent, err := q.StepCalleeParent(ctx, start.CalledExecutionID) + if errors.Is(err, pgx.ErrNoRows) { + return storecontract.ErrStaleTransition + } + if err != nil { + return err + } + if calleeParent != start.ExecutionID { + return storecontract.ErrStaleTransition + } + return nil +} + // stepParent resolves the enclosing StepExecution a child step names: its // row is locked in the same transaction, the tree must stay inside one // execution and one attempt, and depth is derived from the parent, never @@ -258,6 +283,15 @@ func (s *Store) CancelExecution(ctx context.Context, taskID int64, reason, to st return nil, fmt.Errorf("%w: %s -> %s is not a transition", storecontract.ErrIllegalTransition, execution.Status, to) } + // Read the wait:true callees before the sweep below closes them: an open + // call step is a callee the caller waits on, and one whose call step + // already closed -- wait:false, or one that ended first -- runs on. + waiting, err := q.WaitingCallSteps(ctx, postgresdb.WaitingCallStepsParams{ + ExecutionID: taskID, Attempt: execution.Attempt, + }) + if err != nil { + return nil, err + } steps, err := q.CancelAttemptSteps(ctx, postgresdb.CancelAttemptStepsParams{ ExecutionID: taskID, Attempt: execution.Attempt, @@ -295,5 +329,25 @@ func (s *Store) CancelExecution(ctx context.Context, taskID int64, reason, to st for _, step := range steps { ids = append(ids, step.ID) } + + if err := s.cancelWaitedCallees(ctx, waiting, reason, to); err != nil { + return ids, err + } return ids, nil } + +// cancelWaitedCallees cancels each callee the caller was still waiting on +// through its own CancelExecution, under its own lifecycle, in the caller's +// write's aftermath -- the callee's own steps and its own move to the same +// destination. A chain of calls is bounded by the call depth the enqueue +// enforces, so the recursion is finite; a callee that cannot be cancelled is +// reported, and the caller's own record stands either way. +func (s *Store) cancelWaitedCallees(ctx context.Context, waiting []postgresdb.WaitingCallStepsRow, reason, to string) error { + var failures []error + for _, call := range waiting { + if _, err := s.CancelExecution(ctx, call.CalledExecutionID, reason, to); err != nil { + failures = append(failures, fmt.Errorf("cancel callee %d: %w", call.CalledExecutionID, err)) + } + } + return errors.Join(failures...) +} diff --git a/internal/infrastructure/staterpc/client.go b/internal/infrastructure/staterpc/client.go index 63935bc60..5f57be695 100644 --- a/internal/infrastructure/staterpc/client.go +++ b/internal/infrastructure/staterpc/client.go @@ -117,6 +117,7 @@ func (c *Client) StartStep(ctx context.Context, s task.StepStart) (int64, events r, err := c.client.StartStep(ctx, &pb.StartStepRequest{ ExecutionId: s.ExecutionID, Attempt: int64(s.Attempt), ParentId: s.ParentID, Kind: s.Kind, Name: s.Name, + CalledExecutionId: s.CalledExecutionID, }) if err != nil { return 0, events.Event{}, unmapError(err) diff --git a/internal/infrastructure/staterpc/server.go b/internal/infrastructure/staterpc/server.go index 2b18b0736..8f04e5de6 100644 --- a/internal/infrastructure/staterpc/server.go +++ b/internal/infrastructure/staterpc/server.go @@ -211,6 +211,7 @@ func (s *server) StartStep(ctx context.Context, r *pb.StartStepRequest) (*pb.Sta stepID, event, err := s.deps.Steps.StartStep(ctx, task.StepStart{ ExecutionID: r.ExecutionId, Attempt: int(r.Attempt), ParentID: r.ParentId, Kind: r.Kind, Name: r.Name, + CalledExecutionID: r.CalledExecutionId, }) if err != nil { return nil, s.logErr("StartStep", err) diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index ace40c102..722be7fe7 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -537,6 +537,9 @@ message StartStepRequest { // kind is stage, agent or call. string kind = 4; string name = 5; + // called_execution_id names the WorkflowExecution a `call` step started; + // the store verifies it is this execution's own callee. + int64 called_execution_id = 6; } message StartStepResponse { int64 step_id = 1;