feat(statestore): RecoverStale interrupts the steps a crash left running
Plan item 6 of docs/prds/execution-tree-state-machine.md (archie-core-1214), the crash-recovery sweep the step table made decidable. RecoverStale used to be one bare UPDATE of tasks running -> queued; the steps an interrupted execution left running stayed running forever, so the next attempt started under steps that were never closed. Recovery is now one transaction: the running executions are locked, every step each one's current attempt left running moves to the step transition table's interrupted outcome with one stage_finish event row carrying the interruption marker -- the same document stageOutcome reads, so the attempt view renders the interruption the way the pre-step-table producer wrote it -- each execution's audit row lands in the same write, and the existing running->queued requeue query runs last inside it. Both edges are pinned in SQL and pinned to their tables (the execution pair beside claim and BeginRemediation, the step pair in its own test, because the two tables are different tables), so a query edit cannot move a write off a table. Deliberately running-only, per the PRD's letter: pending rows never persist outside StartStep's own transaction, so a crashed attempt leaves only running rows, and a finished step is never rewritten. The interrupted steps' events render as the PRD's interrupted -- distinct from cancelled, which an operator or parent asked for, and from failed, which the step reported. The wait:true-callee and agent-child sweeps ride #1216's tree, and the dashboard's ListSteps read is plan item 7; both untouched here, and the RecoverStale signature and RPC are unchanged.
This commit is contained in:
5 files changed
+253
-4
No files matched your search
@@ -192,3 +192,83 @@ func TestCancelExecutionOnAMissingExecutionIsStale(t *testing.T) {
|
||||
t.Fatalf("CancelExecution on a missing execution = %v, want ErrStaleTransition", err)
|
||||
}
|
||||
}
|
||||
|
||||
// RecoverStale is the crash-recovery half of
|
||||
// docs/prds/execution-tree-state-machine.md: requeueing a running execution
|
||||
// moves the steps its interrupted attempt left running to the table's
|
||||
// interrupted outcome, through the step transition table (the pair pinned in
|
||||
// SQL, like the execution's own edge), with one event row per step transition
|
||||
// and the execution's audit row in the same transaction. Killing the daemon
|
||||
// mid-stage and restarting leaves that stage interrupted and the execution
|
||||
// queued with a fresh attempt for the next run.
|
||||
func TestRecoverStaleInterruptsRunningSteps(t *testing.T) {
|
||||
s := storeFor(t)
|
||||
execution := runningExecution(t, s)
|
||||
|
||||
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
|
||||
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StartStep stage: %v", err)
|
||||
}
|
||||
childID, _, err := s.StartStep(t.Context(), task.StepStart{
|
||||
ExecutionID: execution.ID, Attempt: execution.Attempt,
|
||||
ParentID: stageID, Kind: task.StepKindAgent, Name: "implement",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StartStep child: %v", err)
|
||||
}
|
||||
if _, err := s.FinishStep(t.Context(), task.StepFinish{
|
||||
StepID: stageID, ExecutionID: execution.ID,
|
||||
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done",
|
||||
}); err != nil {
|
||||
t.Fatalf("FinishStep: %v", err)
|
||||
}
|
||||
|
||||
requeued, err := s.RecoverStale(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("RecoverStale: %v", err)
|
||||
}
|
||||
if requeued != 1 {
|
||||
t.Fatalf("RecoverStale requeued %d, want the one running execution", requeued)
|
||||
}
|
||||
|
||||
got, err := s.TaskByID(t.Context(), execution.ID)
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("TaskByID: %+v %v", got, err)
|
||||
}
|
||||
if got.Status != taskstate.Queued {
|
||||
t.Fatalf("execution after recovery = %q, want queued for the next attempt", got.Status)
|
||||
}
|
||||
|
||||
var statuses map[int64]string
|
||||
if err := s.pool.QueryRow(t.Context(),
|
||||
`SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`,
|
||||
execution.ID).Scan(&statuses); err != nil {
|
||||
t.Fatalf("read steps: %v", err)
|
||||
}
|
||||
if statuses[stageID] != string(taskstate.StepSucceeded) {
|
||||
t.Errorf("the finished step was rewritten to %q; recovery only interrupts", statuses[stageID])
|
||||
}
|
||||
if statuses[childID] != string(taskstate.StepInterrupted) {
|
||||
t.Errorf("the step the crash left running = %q, want interrupted", statuses[childID])
|
||||
}
|
||||
|
||||
evs, err := s.TaskEvents(t.Context(), execution.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("TaskEvents: %v", err)
|
||||
}
|
||||
var interrupted []events.Event
|
||||
for _, e := range evs {
|
||||
if e.Data["interrupted"] == true {
|
||||
interrupted = append(interrupted, e)
|
||||
}
|
||||
}
|
||||
if len(interrupted) != 1 {
|
||||
t.Fatalf("interrupted step events = %d, want exactly one for the step transition", len(interrupted))
|
||||
}
|
||||
if interrupted[0].Kind != events.KindStageFinish || interrupted[0].Stage != "implement" ||
|
||||
interrupted[0].Attempt != got.Attempt {
|
||||
t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0])
|
||||
}
|
||||
}
|
||||
@@ -115,6 +115,48 @@ func (q *Queries) InsertStepExecution(ctx context.Context, arg InsertStepExecuti
|
||||
return id, err
|
||||
}
|
||||
|
||||
const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many
|
||||
UPDATE step_executions
|
||||
SET status = 'interrupted', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
|
||||
RETURNING id, name
|
||||
`
|
||||
|
||||
type InterruptAttemptStepsParams struct {
|
||||
ExecutionID int64
|
||||
Attempt int64
|
||||
}
|
||||
|
||||
type InterruptAttemptStepsRow struct {
|
||||
ID int64
|
||||
Name string
|
||||
}
|
||||
|
||||
// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
|
||||
// the steps an interrupted execution left running move to the table's
|
||||
// interrupted outcome, guarded by the row's own status so only the pair the
|
||||
// step transition table routes is written. Returning them lets the store
|
||||
// write one event row per transition in the same transaction.
|
||||
func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemptStepsParams) ([]InterruptAttemptStepsRow, error) {
|
||||
rows, err := q.db.Query(ctx, interruptAttemptSteps, arg.ExecutionID, arg.Attempt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []InterruptAttemptStepsRow
|
||||
for rows.Next() {
|
||||
var i InterruptAttemptStepsRow
|
||||
if err := rows.Scan(&i.ID, &i.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockExecutionForStep = `-- name: LockExecutionForStep :one
|
||||
|
||||
SELECT id, status, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
@@ -158,6 +200,54 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockRunningExecutions = `-- name: LockRunningExecutions :many
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
FROM tasks WHERE status = 'running' FOR UPDATE
|
||||
`
|
||||
|
||||
type LockRunningExecutionsRow struct {
|
||||
ID int64
|
||||
OrgID string
|
||||
WorkspaceID string
|
||||
Owner string
|
||||
Repo string
|
||||
IssueNumber int64
|
||||
Workflow string
|
||||
Attempt int64
|
||||
}
|
||||
|
||||
// The executions a crashed or replaced daemon left running, locked for the
|
||||
// recovery transaction so the step sweep and the requeue cannot race a
|
||||
// concurrent write. The row carries the identity the steps' events need.
|
||||
func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecutionsRow, error) {
|
||||
rows, err := q.db.Query(ctx, lockRunningExecutions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []LockRunningExecutionsRow
|
||||
for rows.Next() {
|
||||
var i LockRunningExecutionsRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.OrgID,
|
||||
&i.WorkspaceID,
|
||||
&i.Owner,
|
||||
&i.Repo,
|
||||
&i.IssueNumber,
|
||||
&i.Workflow,
|
||||
&i.Attempt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockStepExecution = `-- name: LockStepExecution :one
|
||||
SELECT id, org_id, workspace_id, execution_id, attempt, parent_id, depth, kind, name, called_execution_id, status, detail, tokens_used, started_at, finished_at
|
||||
FROM step_executions WHERE id = $1 FOR UPDATE
|
||||
|
||||
@@ -43,3 +43,21 @@ UPDATE step_executions
|
||||
SET status = 'cancelled', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status IN ('pending', 'running')
|
||||
RETURNING id, name;
|
||||
|
||||
-- name: LockRunningExecutions :many
|
||||
-- The executions a crashed or replaced daemon left running, locked for the
|
||||
-- recovery transaction so the step sweep and the requeue cannot race a
|
||||
-- concurrent write. The row carries the identity the steps' events need.
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
FROM tasks WHERE status = 'running' FOR UPDATE;
|
||||
|
||||
-- name: InterruptAttemptSteps :many
|
||||
-- The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
|
||||
-- the steps an interrupted execution left running move to the table's
|
||||
-- interrupted outcome, guarded by the row's own status so only the pair the
|
||||
-- step transition table routes is written. Returning them lets the store
|
||||
-- write one event row per transition in the same transaction.
|
||||
UPDATE step_executions
|
||||
SET status = 'interrupted', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
|
||||
RETURNING id, name;
|
||||
@@ -440,11 +440,56 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
|
||||
return eventID, nil
|
||||
}
|
||||
|
||||
// RecoverStale re-queues tasks left running by a crashed daemon. The write is
|
||||
// the table's crash-recovery edge (running->queued), pinned in SQL and pinned
|
||||
// to the table by TestSQLPinnedTransitionsAreTableLegal.
|
||||
// RecoverStale re-queues tasks left running by a crashed or replaced daemon
|
||||
// and, in the same transaction, moves the steps those executions left running
|
||||
// to the step transition table's interrupted outcome
|
||||
// (docs/prds/execution-tree-state-machine.md, "Crash recovery"): one event row
|
||||
// per step transition and the execution's audit row land in the same write.
|
||||
// Both edges are pinned in SQL (the execution's running->queued, the step's
|
||||
// running->interrupted) and pinned to the tables by
|
||||
// TestSQLPinnedTransitionsAreTableLegal. The next attempt starts with fresh
|
||||
// steps; earlier attempts are never rewritten.
|
||||
func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
|
||||
return s.queries().RecoverStaleTasks(ctx)
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
q := postgresdb.New(tx)
|
||||
|
||||
executions, err := q.LockRunningExecutions(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, execution := range executions {
|
||||
steps, err := q.InterruptAttemptSteps(ctx, postgresdb.InterruptAttemptStepsParams{
|
||||
ExecutionID: execution.ID, Attempt: execution.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, step := range steps {
|
||||
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
|
||||
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
|
||||
int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
|
||||
TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued,
|
||||
Detail: "re-queued after a crashed or replaced daemon",
|
||||
}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n, err := q.RecoverStaleTasks(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// OpenPRs returns tasks whose PR state should be reconciled with the forge,
|
||||
|
||||
@@ -131,6 +131,22 @@ func TestLegalTransitionStillWrites(t *testing.T) {
|
||||
// supplied by a caller (claim, BeginRemediation, RecoverStale) can only write
|
||||
// the pairs asserted here. Pinning them to the shared table keeps a query
|
||||
// edit from moving a write off the table without this file failing.
|
||||
// The step edge RecoverStale writes is table-routed too: the sweep's WHERE
|
||||
// clause pins status = 'running' and writes 'interrupted', so the pair must
|
||||
// stay legal on the STEP table for a query edit to stay on it. The step table
|
||||
// is a different table from the execution's, so it is pinned beside it rather
|
||||
// than mixed into the execution pairs above.
|
||||
func TestSQLPinnedStepTransitionsAreTableLegal(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
from, to taskstate.StepStatus
|
||||
}{{name: "crash recovery pins step running->interrupted", from: taskstate.StepRunning, to: taskstate.StepInterrupted}} {
|
||||
if !taskstate.CanStepTransition(tc.from, tc.to) {
|
||||
t.Errorf("%s: %s -> %s is not in the shared step transition table", tc.name, tc.from, tc.to)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLPinnedTransitionsAreTableLegal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in new issue
Block a user