feat(statestore): RecoverStale interrupts the steps a crash left running

Plan item 6 of docs/prds/execution-tree-state-machine.md (archie-core-1214),
the crash-recovery sweep the step table made decidable. RecoverStale used to be
one bare UPDATE of tasks running -> queued; the steps an interrupted execution
left running stayed running forever, so the next attempt started under steps
that were never closed.

Recovery is now one transaction: the running executions are locked, every step
each one's current attempt left running moves to the step transition table's
interrupted outcome with one stage_finish event row carrying the
interruption marker -- the same document stageOutcome reads, so the attempt
view renders the interruption the way the pre-step-table producer wrote it --
each execution's audit row lands in the same write, and the existing
running->queued requeue query runs last inside it. Both edges are pinned in
SQL and pinned to their tables (the execution pair beside claim and
BeginRemediation, the step pair in its own test, because the two tables are
different tables), so a query edit cannot move a write off a table.

Deliberately running-only, per the PRD's letter: pending rows never persist
outside StartStep's own transaction, so a crashed attempt leaves only running
rows, and a finished step is never rewritten. The interrupted steps' events
render as the PRD's interrupted -- distinct from cancelled, which an operator
or parent asked for, and from failed, which the step reported.

The wait:true-callee and agent-child sweeps ride #1216's tree, and the
dashboard's ListSteps read is plan item 7; both untouched here, and the
RecoverStale signature and RPC are unchanged.
This commit is contained in:
sam committed 2026-09-27 06:47:29 +10:00
1 parent ac5fc4030d
commit 1470a62b9b
5 files changed
+253 -4

No files matched your search

@@ -192,3 +192,83 @@ func TestCancelExecutionOnAMissingExecutionIsStale(t *testing.T) {
t.Fatalf("CancelExecution on a missing execution = %v, want ErrStaleTransition", err)
}
}
// RecoverStale is the crash-recovery half of
// docs/prds/execution-tree-state-machine.md: requeueing a running execution
// moves the steps its interrupted attempt left running to the table's
// interrupted outcome, through the step transition table (the pair pinned in
// SQL, like the execution's own edge), with one event row per step transition
// and the execution's audit row in the same transaction. Killing the daemon
// mid-stage and restarting leaves that stage interrupted and the execution
// queued with a fresh attempt for the next run.
func TestRecoverStaleInterruptsRunningSteps(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep stage: %v", err)
}
childID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stageID, Kind: task.StepKindAgent, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep child: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: stageID, ExecutionID: execution.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done",
}); err != nil {
t.Fatalf("FinishStep: %v", err)
}
requeued, err := s.RecoverStale(t.Context())
if err != nil {
t.Fatalf("RecoverStale: %v", err)
}
if requeued != 1 {
t.Fatalf("RecoverStale requeued %d, want the one running execution", requeued)
}
got, err := s.TaskByID(t.Context(), execution.ID)
if err != nil || got == nil {
t.Fatalf("TaskByID: %+v %v", got, err)
}
if got.Status != taskstate.Queued {
t.Fatalf("execution after recovery = %q, want queued for the next attempt", got.Status)
}
var statuses map[int64]string
if err := s.pool.QueryRow(t.Context(),
`SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`,
execution.ID).Scan(&statuses); err != nil {
t.Fatalf("read steps: %v", err)
}
if statuses[stageID] != string(taskstate.StepSucceeded) {
t.Errorf("the finished step was rewritten to %q; recovery only interrupts", statuses[stageID])
}
if statuses[childID] != string(taskstate.StepInterrupted) {
t.Errorf("the step the crash left running = %q, want interrupted", statuses[childID])
}
evs, err := s.TaskEvents(t.Context(), execution.ID)
if err != nil {
t.Fatalf("TaskEvents: %v", err)
}
var interrupted []events.Event
for _, e := range evs {
if e.Data["interrupted"] == true {
interrupted = append(interrupted, e)
}
}
if len(interrupted) != 1 {
t.Fatalf("interrupted step events = %d, want exactly one for the step transition", len(interrupted))
}
if interrupted[0].Kind != events.KindStageFinish || interrupted[0].Stage != "implement" ||
interrupted[0].Attempt != got.Attempt {
t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0])
}
}
@@ -115,6 +115,48 @@ func (q *Queries) InsertStepExecution(ctx context.Context, arg InsertStepExecuti
return id, err
}
const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many
UPDATE step_executions
SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name
`
type InterruptAttemptStepsParams struct {
ExecutionID int64
Attempt int64
}
type InterruptAttemptStepsRow struct {
ID int64
Name string
}
// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
// the steps an interrupted execution left running move to the table's
// interrupted outcome, guarded by the row's own status so only the pair the
// step transition table routes is written. Returning them lets the store
// write one event row per transition in the same transaction.
func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemptStepsParams) ([]InterruptAttemptStepsRow, error) {
rows, err := q.db.Query(ctx, interruptAttemptSteps, arg.ExecutionID, arg.Attempt)
if err != nil {
return nil, err
}
defer rows.Close()
var items []InterruptAttemptStepsRow
for rows.Next() {
var i InterruptAttemptStepsRow
if err := rows.Scan(&i.ID, &i.Name); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockExecutionForStep = `-- name: LockExecutionForStep :one
SELECT id, status, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
@@ -158,6 +200,54 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu
return i, err
}
const lockRunningExecutions = `-- name: LockRunningExecutions :many
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
FROM tasks WHERE status = 'running' FOR UPDATE
`
type LockRunningExecutionsRow struct {
ID int64
OrgID string
WorkspaceID string
Owner string
Repo string
IssueNumber int64
Workflow string
Attempt int64
}
// The executions a crashed or replaced daemon left running, locked for the
// recovery transaction so the step sweep and the requeue cannot race a
// concurrent write. The row carries the identity the steps' events need.
func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecutionsRow, error) {
rows, err := q.db.Query(ctx, lockRunningExecutions)
if err != nil {
return nil, err
}
defer rows.Close()
var items []LockRunningExecutionsRow
for rows.Next() {
var i LockRunningExecutionsRow
if err := rows.Scan(
&i.ID,
&i.OrgID,
&i.WorkspaceID,
&i.Owner,
&i.Repo,
&i.IssueNumber,
&i.Workflow,
&i.Attempt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockStepExecution = `-- name: LockStepExecution :one
SELECT id, org_id, workspace_id, execution_id, attempt, parent_id, depth, kind, name, called_execution_id, status, detail, tokens_used, started_at, finished_at
FROM step_executions WHERE id = $1 FOR UPDATE
@@ -43,3 +43,21 @@ UPDATE step_executions
SET status = 'cancelled', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status IN ('pending', 'running')
RETURNING id, name;
-- name: LockRunningExecutions :many
-- The executions a crashed or replaced daemon left running, locked for the
-- recovery transaction so the step sweep and the requeue cannot race a
-- concurrent write. The row carries the identity the steps' events need.
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
FROM tasks WHERE status = 'running' FOR UPDATE;
-- name: InterruptAttemptSteps :many
-- The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
-- the steps an interrupted execution left running move to the table's
-- interrupted outcome, guarded by the row's own status so only the pair the
-- step transition table routes is written. Returning them lets the store
-- write one event row per transition in the same transaction.
UPDATE step_executions
SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name;
+49 -4
View File
@@ -440,11 +440,56 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
return eventID, nil
}
// RecoverStale re-queues tasks left running by a crashed daemon. The write is
// the table's crash-recovery edge (running->queued), pinned in SQL and pinned
// to the table by TestSQLPinnedTransitionsAreTableLegal.
// RecoverStale re-queues tasks left running by a crashed or replaced daemon
// and, in the same transaction, moves the steps those executions left running
// to the step transition table's interrupted outcome
// (docs/prds/execution-tree-state-machine.md, "Crash recovery"): one event row
// per step transition and the execution's audit row land in the same write.
// Both edges are pinned in SQL (the execution's running->queued, the step's
// running->interrupted) and pinned to the tables by
// TestSQLPinnedTransitionsAreTableLegal. The next attempt starts with fresh
// steps; earlier attempts are never rewritten.
func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
return s.queries().RecoverStaleTasks(ctx)
tx, err := s.pool.Begin(ctx)
if err != nil {
return 0, err
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
executions, err := q.LockRunningExecutions(ctx)
if err != nil {
return 0, err
}
for _, execution := range executions {
steps, err := q.InterruptAttemptSteps(ctx, postgresdb.InterruptAttemptStepsParams{
ExecutionID: execution.ID, Attempt: execution.Attempt,
})
if err != nil {
return 0, err
}
for _, step := range steps {
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil {
return 0, err
}
}
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued,
Detail: "re-queued after a crashed or replaced daemon",
}); err != nil {
return 0, err
}
}
n, err := q.RecoverStaleTasks(ctx)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, err
}
return n, nil
}
// OpenPRs returns tasks whose PR state should be reconciled with the forge,
@@ -131,6 +131,22 @@ func TestLegalTransitionStillWrites(t *testing.T) {
// supplied by a caller (claim, BeginRemediation, RecoverStale) can only write
// the pairs asserted here. Pinning them to the shared table keeps a query
// edit from moving a write off the table without this file failing.
// The step edge RecoverStale writes is table-routed too: the sweep's WHERE
// clause pins status = 'running' and writes 'interrupted', so the pair must
// stay legal on the STEP table for a query edit to stay on it. The step table
// is a different table from the execution's, so it is pinned beside it rather
// than mixed into the execution pairs above.
func TestSQLPinnedStepTransitionsAreTableLegal(t *testing.T) {
for _, tc := range []struct {
name string
from, to taskstate.StepStatus
}{{name: "crash recovery pins step running->interrupted", from: taskstate.StepRunning, to: taskstate.StepInterrupted}} {
if !taskstate.CanStepTransition(tc.from, tc.to) {
t.Errorf("%s: %s -> %s is not in the shared step transition table", tc.name, tc.from, tc.to)
}
}
}
func TestSQLPinnedTransitionsAreTableLegal(t *testing.T) {
tests := []struct {
name string