diff --git a/internal/infrastructure/postgres/cancel_test.go b/internal/infrastructure/postgres/cancel_test.go index 9cc015f4e..dfa3947d6 100644 --- a/internal/infrastructure/postgres/cancel_test.go +++ b/internal/infrastructure/postgres/cancel_test.go @@ -192,3 +192,83 @@ func TestCancelExecutionOnAMissingExecutionIsStale(t *testing.T) { t.Fatalf("CancelExecution on a missing execution = %v, want ErrStaleTransition", err) } } + +// RecoverStale is the crash-recovery half of +// docs/prds/execution-tree-state-machine.md: requeueing a running execution +// moves the steps its interrupted attempt left running to the table's +// interrupted outcome, through the step transition table (the pair pinned in +// SQL, like the execution's own edge), with one event row per step transition +// and the execution's audit row in the same transaction. Killing the daemon +// mid-stage and restarting leaves that stage interrupted and the execution +// queued with a fresh attempt for the next run. +func TestRecoverStaleInterruptsRunningSteps(t *testing.T) { + s := storeFor(t) + execution := runningExecution(t, s) + + stageID, _, err := s.StartStep(t.Context(), task.StepStart{ + ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement", + }) + if err != nil { + t.Fatalf("StartStep stage: %v", err) + } + childID, _, err := s.StartStep(t.Context(), task.StepStart{ + ExecutionID: execution.ID, Attempt: execution.Attempt, + ParentID: stageID, Kind: task.StepKindAgent, Name: "implement", + }) + if err != nil { + t.Fatalf("StartStep child: %v", err) + } + if _, err := s.FinishStep(t.Context(), task.StepFinish{ + StepID: stageID, ExecutionID: execution.ID, + From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done", + }); err != nil { + t.Fatalf("FinishStep: %v", err) + } + + requeued, err := s.RecoverStale(t.Context()) + if err != nil { + t.Fatalf("RecoverStale: %v", err) + } + if requeued != 1 { + t.Fatalf("RecoverStale requeued %d, want the one running execution", requeued) + } + + got, err := s.TaskByID(t.Context(), execution.ID) + if err != nil || got == nil { + t.Fatalf("TaskByID: %+v %v", got, err) + } + if got.Status != taskstate.Queued { + t.Fatalf("execution after recovery = %q, want queued for the next attempt", got.Status) + } + + var statuses map[int64]string + if err := s.pool.QueryRow(t.Context(), + `SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`, + execution.ID).Scan(&statuses); err != nil { + t.Fatalf("read steps: %v", err) + } + if statuses[stageID] != string(taskstate.StepSucceeded) { + t.Errorf("the finished step was rewritten to %q; recovery only interrupts", statuses[stageID]) + } + if statuses[childID] != string(taskstate.StepInterrupted) { + t.Errorf("the step the crash left running = %q, want interrupted", statuses[childID]) + } + + evs, err := s.TaskEvents(t.Context(), execution.ID) + if err != nil { + t.Fatalf("TaskEvents: %v", err) + } + var interrupted []events.Event + for _, e := range evs { + if e.Data["interrupted"] == true { + interrupted = append(interrupted, e) + } + } + if len(interrupted) != 1 { + t.Fatalf("interrupted step events = %d, want exactly one for the step transition", len(interrupted)) + } + if interrupted[0].Kind != events.KindStageFinish || interrupted[0].Stage != "implement" || + interrupted[0].Attempt != got.Attempt { + t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0]) + } +} diff --git a/internal/infrastructure/postgres/postgresdb/steps.sql.go b/internal/infrastructure/postgres/postgresdb/steps.sql.go index 47ffa89cb..296994dbf 100644 --- a/internal/infrastructure/postgres/postgresdb/steps.sql.go +++ b/internal/infrastructure/postgres/postgresdb/steps.sql.go @@ -115,6 +115,48 @@ func (q *Queries) InsertStepExecution(ctx context.Context, arg InsertStepExecuti return id, err } +const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many +UPDATE step_executions +SET status = 'interrupted', finished_at = now() +WHERE execution_id = $1 AND attempt = $2 AND status = 'running' +RETURNING id, name +` + +type InterruptAttemptStepsParams struct { + ExecutionID int64 + Attempt int64 +} + +type InterruptAttemptStepsRow struct { + ID int64 + Name string +} + +// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md): +// the steps an interrupted execution left running move to the table's +// interrupted outcome, guarded by the row's own status so only the pair the +// step transition table routes is written. Returning them lets the store +// write one event row per transition in the same transaction. +func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemptStepsParams) ([]InterruptAttemptStepsRow, error) { + rows, err := q.db.Query(ctx, interruptAttemptSteps, arg.ExecutionID, arg.Attempt) + if err != nil { + return nil, err + } + defer rows.Close() + var items []InterruptAttemptStepsRow + for rows.Next() { + var i InterruptAttemptStepsRow + if err := rows.Scan(&i.ID, &i.Name); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const lockExecutionForStep = `-- name: LockExecutionForStep :one SELECT id, status, org_id, workspace_id, owner, repo, issue_number, workflow, attempt @@ -158,6 +200,54 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu return i, err } +const lockRunningExecutions = `-- name: LockRunningExecutions :many +SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt +FROM tasks WHERE status = 'running' FOR UPDATE +` + +type LockRunningExecutionsRow struct { + ID int64 + OrgID string + WorkspaceID string + Owner string + Repo string + IssueNumber int64 + Workflow string + Attempt int64 +} + +// The executions a crashed or replaced daemon left running, locked for the +// recovery transaction so the step sweep and the requeue cannot race a +// concurrent write. The row carries the identity the steps' events need. +func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecutionsRow, error) { + rows, err := q.db.Query(ctx, lockRunningExecutions) + if err != nil { + return nil, err + } + defer rows.Close() + var items []LockRunningExecutionsRow + for rows.Next() { + var i LockRunningExecutionsRow + if err := rows.Scan( + &i.ID, + &i.OrgID, + &i.WorkspaceID, + &i.Owner, + &i.Repo, + &i.IssueNumber, + &i.Workflow, + &i.Attempt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const lockStepExecution = `-- name: LockStepExecution :one SELECT id, org_id, workspace_id, execution_id, attempt, parent_id, depth, kind, name, called_execution_id, status, detail, tokens_used, started_at, finished_at FROM step_executions WHERE id = $1 FOR UPDATE diff --git a/internal/infrastructure/postgres/queries/steps.sql b/internal/infrastructure/postgres/queries/steps.sql index d440266f7..0d65260a2 100644 --- a/internal/infrastructure/postgres/queries/steps.sql +++ b/internal/infrastructure/postgres/queries/steps.sql @@ -43,3 +43,21 @@ UPDATE step_executions SET status = 'cancelled', finished_at = now() WHERE execution_id = $1 AND attempt = $2 AND status IN ('pending', 'running') RETURNING id, name; + +-- name: LockRunningExecutions :many +-- The executions a crashed or replaced daemon left running, locked for the +-- recovery transaction so the step sweep and the requeue cannot race a +-- concurrent write. The row carries the identity the steps' events need. +SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt +FROM tasks WHERE status = 'running' FOR UPDATE; + +-- name: InterruptAttemptSteps :many +-- The crash-recovery step edge (docs/prds/execution-tree-state-machine.md): +-- the steps an interrupted execution left running move to the table's +-- interrupted outcome, guarded by the row's own status so only the pair the +-- step transition table routes is written. Returning them lets the store +-- write one event row per transition in the same transaction. +UPDATE step_executions +SET status = 'interrupted', finished_at = now() +WHERE execution_id = $1 AND attempt = $2 AND status = 'running' +RETURNING id, name; diff --git a/internal/infrastructure/postgres/store.go b/internal/infrastructure/postgres/store.go index 985ca1dfa..41c7af997 100644 --- a/internal/infrastructure/postgres/store.go +++ b/internal/infrastructure/postgres/store.go @@ -440,11 +440,56 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string return eventID, nil } -// RecoverStale re-queues tasks left running by a crashed daemon. The write is -// the table's crash-recovery edge (running->queued), pinned in SQL and pinned -// to the table by TestSQLPinnedTransitionsAreTableLegal. +// RecoverStale re-queues tasks left running by a crashed or replaced daemon +// and, in the same transaction, moves the steps those executions left running +// to the step transition table's interrupted outcome +// (docs/prds/execution-tree-state-machine.md, "Crash recovery"): one event row +// per step transition and the execution's audit row land in the same write. +// Both edges are pinned in SQL (the execution's running->queued, the step's +// running->interrupted) and pinned to the tables by +// TestSQLPinnedTransitionsAreTableLegal. The next attempt starts with fresh +// steps; earlier attempts are never rewritten. func (s *Store) RecoverStale(ctx context.Context) (int64, error) { - return s.queries().RecoverStaleTasks(ctx) + tx, err := s.pool.Begin(ctx) + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback(ctx) }() + q := postgresdb.New(tx) + + executions, err := q.LockRunningExecutions(ctx) + if err != nil { + return 0, err + } + for _, execution := range executions { + steps, err := q.InterruptAttemptSteps(ctx, postgresdb.InterruptAttemptStepsParams{ + ExecutionID: execution.ID, Attempt: execution.Attempt, + }) + if err != nil { + return 0, err + } + for _, step := range steps { + if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID, + execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow, + int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil { + return 0, err + } + } + if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{ + TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued, + Detail: "re-queued after a crashed or replaced daemon", + }); err != nil { + return 0, err + } + } + n, err := q.RecoverStaleTasks(ctx) + if err != nil { + return 0, err + } + if err := tx.Commit(ctx); err != nil { + return 0, err + } + return n, nil } // OpenPRs returns tasks whose PR state should be reconciled with the forge, diff --git a/internal/infrastructure/postgres/transition_enforcement_test.go b/internal/infrastructure/postgres/transition_enforcement_test.go index da2203ad4..2089dcdf8 100644 --- a/internal/infrastructure/postgres/transition_enforcement_test.go +++ b/internal/infrastructure/postgres/transition_enforcement_test.go @@ -131,6 +131,22 @@ func TestLegalTransitionStillWrites(t *testing.T) { // supplied by a caller (claim, BeginRemediation, RecoverStale) can only write // the pairs asserted here. Pinning them to the shared table keeps a query // edit from moving a write off the table without this file failing. +// The step edge RecoverStale writes is table-routed too: the sweep's WHERE +// clause pins status = 'running' and writes 'interrupted', so the pair must +// stay legal on the STEP table for a query edit to stay on it. The step table +// is a different table from the execution's, so it is pinned beside it rather +// than mixed into the execution pairs above. +func TestSQLPinnedStepTransitionsAreTableLegal(t *testing.T) { + for _, tc := range []struct { + name string + from, to taskstate.StepStatus + }{{name: "crash recovery pins step running->interrupted", from: taskstate.StepRunning, to: taskstate.StepInterrupted}} { + if !taskstate.CanStepTransition(tc.from, tc.to) { + t.Errorf("%s: %s -> %s is not in the shared step transition table", tc.name, tc.from, tc.to) + } + } +} + func TestSQLPinnedTransitionsAreTableLegal(t *testing.T) { tests := []struct { name string