Merge crew/pi-dashboard-sweep: RecoverStale interrupts the steps a crash left running

This commit is contained in:
sam committed 2026-09-27 06:51:07 +10:00
commit 44bea32b16
5 files changed
+253 -4

No files matched your search

@@ -192,3 +192,83 @@ func TestCancelExecutionOnAMissingExecutionIsStale(t *testing.T) {
t.Fatalf("CancelExecution on a missing execution = %v, want ErrStaleTransition", err)
}
}
// RecoverStale is the crash-recovery half of
// docs/prds/execution-tree-state-machine.md: requeueing a running execution
// moves the steps its interrupted attempt left running to the table's
// interrupted outcome, through the step transition table (the pair pinned in
// SQL, like the execution's own edge), with one event row per step transition
// and the execution's audit row in the same transaction. Killing the daemon
// mid-stage and restarting leaves that stage interrupted and the execution
// queued with a fresh attempt for the next run.
func TestRecoverStaleInterruptsRunningSteps(t *testing.T) {
s := storeFor(t)
execution := runningExecution(t, s)
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep stage: %v", err)
}
childID, _, err := s.StartStep(t.Context(), task.StepStart{
ExecutionID: execution.ID, Attempt: execution.Attempt,
ParentID: stageID, Kind: task.StepKindAgent, Name: "implement",
})
if err != nil {
t.Fatalf("StartStep child: %v", err)
}
if _, err := s.FinishStep(t.Context(), task.StepFinish{
StepID: stageID, ExecutionID: execution.ID,
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done",
}); err != nil {
t.Fatalf("FinishStep: %v", err)
}
requeued, err := s.RecoverStale(t.Context())
if err != nil {
t.Fatalf("RecoverStale: %v", err)
}
if requeued != 1 {
t.Fatalf("RecoverStale requeued %d, want the one running execution", requeued)
}
got, err := s.TaskByID(t.Context(), execution.ID)
if err != nil || got == nil {
t.Fatalf("TaskByID: %+v %v", got, err)
}
if got.Status != taskstate.Queued {
t.Fatalf("execution after recovery = %q, want queued for the next attempt", got.Status)
}
var statuses map[int64]string
if err := s.pool.QueryRow(t.Context(),
`SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`,
execution.ID).Scan(&statuses); err != nil {
t.Fatalf("read steps: %v", err)
}
if statuses[stageID] != string(taskstate.StepSucceeded) {
t.Errorf("the finished step was rewritten to %q; recovery only interrupts", statuses[stageID])
}
if statuses[childID] != string(taskstate.StepInterrupted) {
t.Errorf("the step the crash left running = %q, want interrupted", statuses[childID])
}
evs, err := s.TaskEvents(t.Context(), execution.ID)
if err != nil {
t.Fatalf("TaskEvents: %v", err)
}
var interrupted []events.Event
for _, e := range evs {
if e.Data["interrupted"] == true {
interrupted = append(interrupted, e)
}
}
if len(interrupted) != 1 {
t.Fatalf("interrupted step events = %d, want exactly one for the step transition", len(interrupted))
}
if interrupted[0].Kind != events.KindStageFinish || interrupted[0].Stage != "implement" ||
interrupted[0].Attempt != got.Attempt {
t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0])
}
}
@@ -115,6 +115,48 @@ func (q *Queries) InsertStepExecution(ctx context.Context, arg InsertStepExecuti
return id, err
}
const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many
UPDATE step_executions
SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name
`
type InterruptAttemptStepsParams struct {
ExecutionID int64
Attempt int64
}
type InterruptAttemptStepsRow struct {
ID int64
Name string
}
// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
// the steps an interrupted execution left running move to the table's
// interrupted outcome, guarded by the row's own status so only the pair the
// step transition table routes is written. Returning them lets the store
// write one event row per transition in the same transaction.
func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemptStepsParams) ([]InterruptAttemptStepsRow, error) {
rows, err := q.db.Query(ctx, interruptAttemptSteps, arg.ExecutionID, arg.Attempt)
if err != nil {
return nil, err
}
defer rows.Close()
var items []InterruptAttemptStepsRow
for rows.Next() {
var i InterruptAttemptStepsRow
if err := rows.Scan(&i.ID, &i.Name); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockExecutionForStep = `-- name: LockExecutionForStep :one
SELECT id, status, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
@@ -158,6 +200,54 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu
return i, err
}
const lockRunningExecutions = `-- name: LockRunningExecutions :many
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
FROM tasks WHERE status = 'running' FOR UPDATE
`
type LockRunningExecutionsRow struct {
ID int64
OrgID string
WorkspaceID string
Owner string
Repo string
IssueNumber int64
Workflow string
Attempt int64
}
// The executions a crashed or replaced daemon left running, locked for the
// recovery transaction so the step sweep and the requeue cannot race a
// concurrent write. The row carries the identity the steps' events need.
func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecutionsRow, error) {
rows, err := q.db.Query(ctx, lockRunningExecutions)
if err != nil {
return nil, err
}
defer rows.Close()
var items []LockRunningExecutionsRow
for rows.Next() {
var i LockRunningExecutionsRow
if err := rows.Scan(
&i.ID,
&i.OrgID,
&i.WorkspaceID,
&i.Owner,
&i.Repo,
&i.IssueNumber,
&i.Workflow,
&i.Attempt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockStepExecution = `-- name: LockStepExecution :one
SELECT id, org_id, workspace_id, execution_id, attempt, parent_id, depth, kind, name, called_execution_id, status, detail, tokens_used, started_at, finished_at
FROM step_executions WHERE id = $1 FOR UPDATE
@@ -43,3 +43,21 @@ UPDATE step_executions
SET status = 'cancelled', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status IN ('pending', 'running')
RETURNING id, name;
-- name: LockRunningExecutions :many
-- The executions a crashed or replaced daemon left running, locked for the
-- recovery transaction so the step sweep and the requeue cannot race a
-- concurrent write. The row carries the identity the steps' events need.
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
FROM tasks WHERE status = 'running' FOR UPDATE;
-- name: InterruptAttemptSteps :many
-- The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
-- the steps an interrupted execution left running move to the table's
-- interrupted outcome, guarded by the row's own status so only the pair the
-- step transition table routes is written. Returning them lets the store
-- write one event row per transition in the same transaction.
UPDATE step_executions
SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name;
+49 -4
View File
@@ -440,11 +440,56 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
return eventID, nil
}
// RecoverStale re-queues tasks left running by a crashed daemon. The write is
// the table's crash-recovery edge (running->queued), pinned in SQL and pinned
// to the table by TestSQLPinnedTransitionsAreTableLegal.
// RecoverStale re-queues tasks left running by a crashed or replaced daemon
// and, in the same transaction, moves the steps those executions left running
// to the step transition table's interrupted outcome
// (docs/prds/execution-tree-state-machine.md, "Crash recovery"): one event row
// per step transition and the execution's audit row land in the same write.
// Both edges are pinned in SQL (the execution's running->queued, the step's
// running->interrupted) and pinned to the tables by
// TestSQLPinnedTransitionsAreTableLegal. The next attempt starts with fresh
// steps; earlier attempts are never rewritten.
func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
return s.queries().RecoverStaleTasks(ctx)
tx, err := s.pool.Begin(ctx)
if err != nil {
return 0, err
}
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
executions, err := q.LockRunningExecutions(ctx)
if err != nil {
return 0, err
}
for _, execution := range executions {
steps, err := q.InterruptAttemptSteps(ctx, postgresdb.InterruptAttemptStepsParams{
ExecutionID: execution.ID, Attempt: execution.Attempt,
})
if err != nil {
return 0, err
}
for _, step := range steps {
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil {
return 0, err
}
}
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued,
Detail: "re-queued after a crashed or replaced daemon",
}); err != nil {
return 0, err
}
}
n, err := q.RecoverStaleTasks(ctx)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, err
}
return n, nil
}
// OpenPRs returns tasks whose PR state should be reconciled with the forge,
@@ -131,6 +131,22 @@ func TestLegalTransitionStillWrites(t *testing.T) {
// supplied by a caller (claim, BeginRemediation, RecoverStale) can only write
// the pairs asserted here. Pinning them to the shared table keeps a query
// edit from moving a write off the table without this file failing.
// The step edge RecoverStale writes is table-routed too: the sweep's WHERE
// clause pins status = 'running' and writes 'interrupted', so the pair must
// stay legal on the STEP table for a query edit to stay on it. The step table
// is a different table from the execution's, so it is pinned beside it rather
// than mixed into the execution pairs above.
func TestSQLPinnedStepTransitionsAreTableLegal(t *testing.T) {
for _, tc := range []struct {
name string
from, to taskstate.StepStatus
}{{name: "crash recovery pins step running->interrupted", from: taskstate.StepRunning, to: taskstate.StepInterrupted}} {
if !taskstate.CanStepTransition(tc.from, tc.to) {
t.Errorf("%s: %s -> %s is not in the shared step transition table", tc.name, tc.from, tc.to)
}
}
}
func TestSQLPinnedTransitionsAreTableLegal(t *testing.T) {
tests := []struct {
name string