Files
archie/internal/infrastructure/postgres/cancel_test.go
T
sam 1686fba12b feat(workflow): agent-call children, call steps, and the waited-on callee sweep
Plan item 4 of docs/prds/execution-tree-state-machine.md (archie-core-1216),
plus the callee half of CancelExecution that plan item 5 deferred until the
tree could decide it.

Every agent call a stage makes records itself: AgentStage's bridge, the
baseline-fix auto-fix run, the adversarial reviewer and a workflow.call each
start a child StepExecution of kind agent or call under the stage's own step
-- TaskContext.StepID, which the engine sets before the stage body starts and
clears when it returns -- and finish it with the call's own outcome and the
tokens it accounted. A child's recording write failure parks the execution
the same way a stage's does: an agent call that ran unrecorded is the
invisibility the step table exists to end. A stage invoked outside the
engine's recorded run -- the stage unit tests -- records no child and runs
unchanged.

The call step is what makes the callee half of the PRD's Cancellation section
decidable, and this lands it: a call step closes when its callee ends, so an
open call step is a callee the caller waits on (wait:true) and a closed one
is a wait:false callee that runs on. CancelExecution reads the open call
steps before its sweep closes them and cancels each named callee through its
own CancelExecution in the caller's write's aftermath -- the callee's own
steps, its own events, its own move to the same destination, and the callee's
own wait:true callees beyond it, the depth the enqueue enforces bounding the
recursion. A callee that cannot be cancelled is reported and the caller's own
record stands.

StartStep refuses a call step whose callee is not this run's own (the
callee's call_parent_task_id must name the recording execution): a
compromised container could otherwise record a step naming a foreign
execution and cancel it. StartStepRequest gains called_execution_id (field 6;
non-breaking, the column and its named consumer already exist) and
StepStart carries it.

PRD decisions recorded here:
- The reviewer records as one agent child named "review": the Reviewer
  contract runs one subagent per review, the PRD model's review:1/review:2
  are illustrative of depth-1 children. A completed report closes it
  succeeded, a not-run review failed with the skip reason; the contract
  carries no usage, so tokens are 0.
- A failed-callee call step closes as failed with the callee's detail when
  the wait ends otherwise; the single open case is the caller itself being
  cancelled, whose CancelExecution already swept this step -- a late write
  must not pretend the wait is still the caller's business.
- A call step's callee is cancelled to the same destination the caller's
  action named (parked stays parked, closed_wont_do ends the whole chain),
  and a callee that cannot be cancelled is reported while the caller's own
  record stands either way.
- The agent child's detail is the result summary on success and the error's
  text on failure, with the call's own TokensUsed -- the PRD's only usage
  field, untracked per call for the reviewer contract.
2026-09-27 07:14:15 +10:00

367 lines
14 KiB
Go

package postgres
import (
"errors"
"testing"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
task "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// CancelExecution is the one cancel path (docs/prds/execution-tree-state-machine.md,
// "Cancellation"): one transaction cancels every non-terminal StepExecution of
// the execution's current attempt and moves the execution to the status the
// operator action names, with one event row per step transition and the
// execution's own audit row in the same write.
func TestCancelExecutionCancelsTheAttemptsStepsAndTheExecution(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "review",
})
if err != nil {
t.Fatalf("StartStep stage: %v", err)
}
childID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stageID, Kind: task.StepKindAgent, Name: "review:1",
})
if err != nil {
t.Fatalf("StartStep child: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: stageID, ExecutionID: execution.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done",
}); err != nil {
t.Fatalf("FinishStep: %v", err)
}
steps, err := s.CancelExecution(t.Context(), execution.ID, "stopped by operator", taskstate.Parked)
if err != nil {
t.Fatalf("CancelExecution: %v", err)
}
if len(steps) != 1 || steps[0] != childID {
t.Fatalf("cancelled steps = %v, want only the non-terminal %d", steps, childID)
}
// Every non-terminal step of the current attempt is cancelled; the one
// that already finished is never rewritten.
var statuses map[int64]string
if err := s.pool.QueryRow(t.Context(),
`SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`,
execution.ID).Scan(&statuses); err != nil {
t.Fatalf("read steps: %v", err)
}
if statuses[stageID] != string(taskstate.StepSucceeded) || statuses[childID] != string(taskstate.StepCancelled) {
t.Fatalf("statuses = %v, want the finished step kept and the child cancelled", statuses)
}
got, err := s.TaskByID(t.Context(), execution.ID)
if err != nil || got == nil {
t.Fatalf("TaskByID: %+v %v", got, err)
}
if got.Status != taskstate.Parked || got.ParkReason != "stopped by operator" {
t.Fatalf("execution after cancel = %s (%q), want parked with the reason", got.Status, got.ParkReason)
}
}
func TestCancelExecutionWritesOneEventPerCancelledStep(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep: %v", err)
}
if _, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stageID, Kind: task.StepKindAgent, Name: "implement",
}); err != nil {
t.Fatalf("StartStep child: %v", err)
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "stopped by operator", taskstate.Parked); err != nil {
t.Fatalf("CancelExecution: %v", err)
}
evs, err := s.TaskEvents(t.Context(), execution.ID)
if err != nil {
t.Fatalf("TaskEvents: %v", err)
}
var cancelled []events.Event
for _, e := range evs {
if e.Data["cancelled"] == true {
cancelled = append(cancelled, e)
}
}
if len(cancelled) != 2 {
t.Fatalf("cancel events = %d, want one per step the transaction cancelled", len(cancelled))
}
for _, e := range cancelled {
if e.Kind != events.KindStageFinish || e.Attempt != execution.Attempt {
t.Errorf("cancel event = %+v, want a stage_finish row attributed to the attempt", e)
}
}
}
func TestCancelExecutionLeavesEarlierAttemptsAlone(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
firstAttempt := execution.Attempt
stepID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: firstAttempt, Kind: task.StepKindStage, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: stepID, ExecutionID: execution.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded,
}); err != nil {
t.Fatalf("FinishStep: %v", err)
}
// A retry requeues and reclaims: attempt 2.
if err := s.Requeue(t.Context(), execution.ID, taskstate.Running, "implement"); err != nil {
t.Fatalf("Requeue: %v", err)
}
retried, err := s.ClaimNext(t.Context())
if err != nil || retried == nil {
t.Fatalf("ClaimNext: %+v %v", retried, err)
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "stopped by operator", taskstate.Parked); err != nil {
t.Fatalf("CancelExecution: %v", err)
}
var firstAttemptStatuses []string
if err := s.pool.QueryRow(t.Context(),
`SELECT jsonb_agg(status) FROM step_executions WHERE execution_id = $1 AND attempt = $2`,
execution.ID, firstAttempt).Scan(&firstAttemptStatuses); err != nil {
t.Fatalf("read first attempt's steps: %v", err)
}
for _, status := range firstAttemptStatuses {
if status != string(taskstate.StepSucceeded) {
t.Errorf("an earlier attempt's step was rewritten to %q; history is never rewritten", status)
}
}
}
func TestCancelExecutionRefusesWhatTheTableForbids(t *testing.T) {
tests := []struct {
name string
target string
want error
}{
{name: "a terminal execution", target: taskstate.Queued, want: storecontract.ErrIllegalTransition},
{name: "an unroutable destination", target: "quantum", want: storecontract.ErrIllegalTransition},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
if tt.name == "a terminal execution" {
// completed is terminal: the table routes nothing out of it.
if err := s.Transition(t.Context(), execution.ID, taskstate.Running, taskstate.Completed, "done"); err != nil {
t.Fatalf("seed completed: %v", err)
}
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "why", tt.target); !errors.Is(err, tt.want) {
t.Fatalf("CancelExecution(to=%q) = %v, want %v", tt.target, err, tt.want)
}
})
}
// A queued execution cancels fine: the table routes queued -> closed_wont_do.
s := storeFor(t)
execution := runningExecution(t, s)
if err := s.Transition(t.Context(), execution.ID, taskstate.Running, taskstate.Queued, "requeue"); err != nil {
t.Fatalf("seed queued: %v", err)
}
if _, err := s.CancelExecution(t.Context(), execution.ID, "cancelled by operator", taskstate.Declined); err != nil {
t.Fatalf("CancelExecution(queued -> closed_wont_do) = %v, want nil", err)
}
}
func TestCancelExecutionOnAMissingExecutionIsStale(t *testing.T) {
s := storeFor(t)
if _, err := s.CancelExecution(t.Context(), 999999, "why", taskstate.Parked); !errors.Is(err, storecontract.ErrStaleTransition) {
t.Fatalf("CancelExecution on a missing execution = %v, want ErrStaleTransition", err)
}
}
// RecoverStale is the crash-recovery half of
// docs/prds/execution-tree-state-machine.md: requeueing a running execution
// moves the steps its interrupted attempt left running to the table's
// interrupted outcome, through the step transition table (the pair pinned in
// SQL, like the execution's own edge), with one event row per step transition
// and the execution's audit row in the same transaction. Killing the daemon
// mid-stage and restarting leaves that stage interrupted and the execution
// queued with a fresh attempt for the next run.
func TestRecoverStaleInterruptsRunningSteps(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep stage: %v", err)
}
childID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stageID, Kind: task.StepKindAgent, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep child: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: stageID, ExecutionID: execution.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done",
}); err != nil {
t.Fatalf("FinishStep: %v", err)
}
requeued, err := s.RecoverStale(t.Context())
if err != nil {
t.Fatalf("RecoverStale: %v", err)
}
if requeued != 1 {
t.Fatalf("RecoverStale requeued %d, want the one running execution", requeued)
}
got, err := s.TaskByID(t.Context(), execution.ID)
if err != nil || got == nil {
t.Fatalf("TaskByID: %+v %v", got, err)
}
if got.Status != taskstate.Queued {
t.Fatalf("execution after recovery = %q, want queued for the next attempt", got.Status)
}
var statuses map[int64]string
if err := s.pool.QueryRow(t.Context(),
`SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`,
execution.ID).Scan(&statuses); err != nil {
t.Fatalf("read steps: %v", err)
}
if statuses[stageID] != string(taskstate.StepSucceeded) {
t.Errorf("the finished step was rewritten to %q; recovery only interrupts", statuses[stageID])
}
if statuses[childID] != string(taskstate.StepInterrupted) {
t.Errorf("the step the crash left running = %q, want interrupted", statuses[childID])
}
evs, err := s.TaskEvents(t.Context(), execution.ID)
if err != nil {
t.Fatalf("TaskEvents: %v", err)
}
var interrupted []events.Event
for _, e := range evs {
if e.Data["interrupted"] == true {
interrupted = append(interrupted, e)
}
}
if len(interrupted) != 1 {
t.Fatalf("interrupted step events = %d, want exactly one for the step transition", len(interrupted))
}
if interrupted[0].Kind != events.KindStageFinish || interrupted[0].Stage != "implement" ||
interrupted[0].Attempt != got.Attempt {
t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0])
}
}
// A wait:true call step is how the store knows the caller is still waiting on
// its callee: it closes only when the callee ends, so a non-terminal call
// step is a callee the caller waits on. Cancelling the caller sweeps each
// such callee through its own CancelExecution -- the callee's own lifecycle
// -- while a finished call step's callee (wait:false, or already ended) runs
// on. Trees never cross runs: a call step naming a task that is not the
// caller's callee cannot even be recorded (StartStep verifies the callee's
// call_parent_task_id).
func TestCancelExecutionCancelsTheCalleesTheCallerWaitsOn(t *testing.T) {
s := storeFor(t)
caller := runningExecution(t, s)
// The callee: started through the call linkage the way a wait:true step
// does (the store derives the callee from the caller's own row), running
// under its own lifecycle, with a running step of its own.
callee, err := s.StartCall(t.Context(), caller.ID, "implement", nil)
if err != nil {
t.Fatalf("EnqueueCallTask: %v", err)
}
if _, err := s.ClaimNext(t.Context()); err != nil {
t.Fatalf("ClaimNext callee: %v", err)
}
// The claim increments the attempt; read the row back the way the
// container's own request carries it.
callee, err = s.TaskByID(t.Context(), callee.ID)
if err != nil || callee == nil {
t.Fatalf("TaskByID(callee): %+v %v", callee, err)
}
if _, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: callee.ID, Attempt: callee.Attempt, Kind: task.StepKindStage, Name: "implement",
}); err != nil {
t.Fatalf("callee StartStep: %v", err)
}
// The caller's wait:true call step names the callee and is still running.
if _, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: caller.ID, Attempt: caller.Attempt, Kind: task.StepKindCall,
Name: "workflow.call", CalledExecutionID: callee.ID,
}); err != nil {
t.Fatalf("call step: %v", err)
}
// A wait:false call step's callee runs on: its call step already ended.
unwaited, err := s.StartCall(t.Context(), caller.ID, "implement", nil)
if err != nil {
t.Fatal(err)
}
if _, err := s.ClaimNext(t.Context()); err != nil {
t.Fatalf("ClaimNext no-wait callee: %v", err)
}
noWaitStep, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: caller.ID, Attempt: caller.Attempt, Kind: task.StepKindCall,
Name: "workflow.call", CalledExecutionID: unwaited.ID,
})
if err != nil {
t.Fatalf("no-wait call step: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: noWaitStep, ExecutionID: caller.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "started; no wait",
}); err != nil {
t.Fatalf("close the no-wait call step: %v", err)
}
if _, err := s.CancelExecution(t.Context(), caller.ID, "stopped by operator", taskstate.Parked); err != nil {
t.Fatalf("CancelExecution: %v", err)
}
waited, err := s.TaskByID(t.Context(), callee.ID)
if err != nil || waited == nil {
t.Fatalf("TaskByID(callee): %+v %v", waited, err)
}
if waited.Status != taskstate.Parked {
t.Fatalf("the waited-on callee = %q, want parked under its own lifecycle", waited.Status)
}
waitedSteps, err := s.TaskEvents(t.Context(), callee.ID)
if err != nil {
t.Fatalf("callee TaskEvents: %v", err)
}
if len(waitedSteps) == 0 {
t.Error("the callee's steps were not cancelled with their events")
}
unwaitedAfter, err := s.TaskByID(t.Context(), unwaited.ID)
if err != nil || unwaitedAfter == nil {
t.Fatalf("TaskByID no-wait callee: %+v %v", unwaitedAfter, err)
}
if unwaitedAfter.Status != taskstate.Running {
t.Errorf("the wait:false callee = %q, want it running on", unwaitedAfter.Status)
}
}