Merge crew/pi-dashboard-sweep: RecoverStale interrupts the steps a crash left running
This commit is contained in:
commit
44bea32b16
5 files changed
+253
-4
No files matched your search
@@ -192,3 +192,83 @@ func TestCancelExecutionOnAMissingExecutionIsStale(t *testing.T) {
|
||||
t.Fatalf("CancelExecution on a missing execution = %v, want ErrStaleTransition", err)
|
||||
}
|
||||
}
|
||||
|
||||
// RecoverStale is the crash-recovery half of
|
||||
// docs/prds/execution-tree-state-machine.md: requeueing a running execution
|
||||
// moves the steps its interrupted attempt left running to the table's
|
||||
// interrupted outcome, through the step transition table (the pair pinned in
|
||||
// SQL, like the execution's own edge), with one event row per step transition
|
||||
// and the execution's audit row in the same transaction. Killing the daemon
|
||||
// mid-stage and restarting leaves that stage interrupted and the execution
|
||||
// queued with a fresh attempt for the next run.
|
||||
func TestRecoverStaleInterruptsRunningSteps(t *testing.T) {
|
||||
s := storeFor(t)
|
||||
execution := runningExecution(t, s)
|
||||
|
||||
stageID, _, err := s.StartStep(t.Context(), task.StepStart{
|
||||
ExecutionID: execution.ID, Attempt: execution.Attempt, Kind: task.StepKindStage, Name: "implement",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StartStep stage: %v", err)
|
||||
}
|
||||
childID, _, err := s.StartStep(t.Context(), task.StepStart{
|
||||
ExecutionID: execution.ID, Attempt: execution.Attempt,
|
||||
ParentID: stageID, Kind: task.StepKindAgent, Name: "implement",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StartStep child: %v", err)
|
||||
}
|
||||
if _, err := s.FinishStep(t.Context(), task.StepFinish{
|
||||
StepID: stageID, ExecutionID: execution.ID,
|
||||
From: taskstate.StepRunning, To: taskstate.StepSucceeded, Detail: "done",
|
||||
}); err != nil {
|
||||
t.Fatalf("FinishStep: %v", err)
|
||||
}
|
||||
|
||||
requeued, err := s.RecoverStale(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("RecoverStale: %v", err)
|
||||
}
|
||||
if requeued != 1 {
|
||||
t.Fatalf("RecoverStale requeued %d, want the one running execution", requeued)
|
||||
}
|
||||
|
||||
got, err := s.TaskByID(t.Context(), execution.ID)
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("TaskByID: %+v %v", got, err)
|
||||
}
|
||||
if got.Status != taskstate.Queued {
|
||||
t.Fatalf("execution after recovery = %q, want queued for the next attempt", got.Status)
|
||||
}
|
||||
|
||||
var statuses map[int64]string
|
||||
if err := s.pool.QueryRow(t.Context(),
|
||||
`SELECT jsonb_object_agg(id, status) FROM step_executions WHERE execution_id = $1`,
|
||||
execution.ID).Scan(&statuses); err != nil {
|
||||
t.Fatalf("read steps: %v", err)
|
||||
}
|
||||
if statuses[stageID] != string(taskstate.StepSucceeded) {
|
||||
t.Errorf("the finished step was rewritten to %q; recovery only interrupts", statuses[stageID])
|
||||
}
|
||||
if statuses[childID] != string(taskstate.StepInterrupted) {
|
||||
t.Errorf("the step the crash left running = %q, want interrupted", statuses[childID])
|
||||
}
|
||||
|
||||
evs, err := s.TaskEvents(t.Context(), execution.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("TaskEvents: %v", err)
|
||||
}
|
||||
var interrupted []events.Event
|
||||
for _, e := range evs {
|
||||
if e.Data["interrupted"] == true {
|
||||
interrupted = append(interrupted, e)
|
||||
}
|
||||
}
|
||||
if len(interrupted) != 1 {
|
||||
t.Fatalf("interrupted step events = %d, want exactly one for the step transition", len(interrupted))
|
||||
}
|
||||
if interrupted[0].Kind != events.KindStageFinish || interrupted[0].Stage != "implement" ||
|
||||
interrupted[0].Attempt != got.Attempt {
|
||||
t.Errorf("interrupted event = %+v, want a stage_finish row attributed to the interrupted attempt", interrupted[0])
|
||||
}
|
||||
}
|
||||
@@ -115,6 +115,48 @@ func (q *Queries) InsertStepExecution(ctx context.Context, arg InsertStepExecuti
|
||||
return id, err
|
||||
}
|
||||
|
||||
const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many
|
||||
UPDATE step_executions
|
||||
SET status = 'interrupted', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
|
||||
RETURNING id, name
|
||||
`
|
||||
|
||||
type InterruptAttemptStepsParams struct {
|
||||
ExecutionID int64
|
||||
Attempt int64
|
||||
}
|
||||
|
||||
type InterruptAttemptStepsRow struct {
|
||||
ID int64
|
||||
Name string
|
||||
}
|
||||
|
||||
// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
|
||||
// the steps an interrupted execution left running move to the table's
|
||||
// interrupted outcome, guarded by the row's own status so only the pair the
|
||||
// step transition table routes is written. Returning them lets the store
|
||||
// write one event row per transition in the same transaction.
|
||||
func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemptStepsParams) ([]InterruptAttemptStepsRow, error) {
|
||||
rows, err := q.db.Query(ctx, interruptAttemptSteps, arg.ExecutionID, arg.Attempt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []InterruptAttemptStepsRow
|
||||
for rows.Next() {
|
||||
var i InterruptAttemptStepsRow
|
||||
if err := rows.Scan(&i.ID, &i.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockExecutionForStep = `-- name: LockExecutionForStep :one
|
||||
|
||||
SELECT id, status, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
@@ -158,6 +200,54 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu
|
||||
return i, err
|
||||
}
|
||||
|
||||
const lockRunningExecutions = `-- name: LockRunningExecutions :many
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
FROM tasks WHERE status = 'running' FOR UPDATE
|
||||
`
|
||||
|
||||
type LockRunningExecutionsRow struct {
|
||||
ID int64
|
||||
OrgID string
|
||||
WorkspaceID string
|
||||
Owner string
|
||||
Repo string
|
||||
IssueNumber int64
|
||||
Workflow string
|
||||
Attempt int64
|
||||
}
|
||||
|
||||
// The executions a crashed or replaced daemon left running, locked for the
|
||||
// recovery transaction so the step sweep and the requeue cannot race a
|
||||
// concurrent write. The row carries the identity the steps' events need.
|
||||
func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecutionsRow, error) {
|
||||
rows, err := q.db.Query(ctx, lockRunningExecutions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []LockRunningExecutionsRow
|
||||
for rows.Next() {
|
||||
var i LockRunningExecutionsRow
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.OrgID,
|
||||
&i.WorkspaceID,
|
||||
&i.Owner,
|
||||
&i.Repo,
|
||||
&i.IssueNumber,
|
||||
&i.Workflow,
|
||||
&i.Attempt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const lockStepExecution = `-- name: LockStepExecution :one
|
||||
SELECT id, org_id, workspace_id, execution_id, attempt, parent_id, depth, kind, name, called_execution_id, status, detail, tokens_used, started_at, finished_at
|
||||
FROM step_executions WHERE id = $1 FOR UPDATE
|
||||
|
||||
@@ -43,3 +43,21 @@ UPDATE step_executions
|
||||
SET status = 'cancelled', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status IN ('pending', 'running')
|
||||
RETURNING id, name;
|
||||
|
||||
-- name: LockRunningExecutions :many
|
||||
-- The executions a crashed or replaced daemon left running, locked for the
|
||||
-- recovery transaction so the step sweep and the requeue cannot race a
|
||||
-- concurrent write. The row carries the identity the steps' events need.
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
FROM tasks WHERE status = 'running' FOR UPDATE;
|
||||
|
||||
-- name: InterruptAttemptSteps :many
|
||||
-- The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
|
||||
-- the steps an interrupted execution left running move to the table's
|
||||
-- interrupted outcome, guarded by the row's own status so only the pair the
|
||||
-- step transition table routes is written. Returning them lets the store
|
||||
-- write one event row per transition in the same transaction.
|
||||
UPDATE step_executions
|
||||
SET status = 'interrupted', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
|
||||
RETURNING id, name;
|
||||
@@ -440,11 +440,56 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
|
||||
return eventID, nil
|
||||
}
|
||||
|
||||
// RecoverStale re-queues tasks left running by a crashed daemon. The write is
|
||||
// the table's crash-recovery edge (running->queued), pinned in SQL and pinned
|
||||
// to the table by TestSQLPinnedTransitionsAreTableLegal.
|
||||
// RecoverStale re-queues tasks left running by a crashed or replaced daemon
|
||||
// and, in the same transaction, moves the steps those executions left running
|
||||
// to the step transition table's interrupted outcome
|
||||
// (docs/prds/execution-tree-state-machine.md, "Crash recovery"): one event row
|
||||
// per step transition and the execution's audit row land in the same write.
|
||||
// Both edges are pinned in SQL (the execution's running->queued, the step's
|
||||
// running->interrupted) and pinned to the tables by
|
||||
// TestSQLPinnedTransitionsAreTableLegal. The next attempt starts with fresh
|
||||
// steps; earlier attempts are never rewritten.
|
||||
func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
|
||||
return s.queries().RecoverStaleTasks(ctx)
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
q := postgresdb.New(tx)
|
||||
|
||||
executions, err := q.LockRunningExecutions(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, execution := range executions {
|
||||
steps, err := q.InterruptAttemptSteps(ctx, postgresdb.InterruptAttemptStepsParams{
|
||||
ExecutionID: execution.ID, Attempt: execution.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, step := range steps {
|
||||
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
|
||||
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
|
||||
int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if err := q.InsertTransition(ctx, postgresdb.InsertTransitionParams{
|
||||
TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued,
|
||||
Detail: "re-queued after a crashed or replaced daemon",
|
||||
}); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n, err := q.RecoverStaleTasks(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// OpenPRs returns tasks whose PR state should be reconciled with the forge,
|
||||
|
||||
@@ -131,6 +131,22 @@ func TestLegalTransitionStillWrites(t *testing.T) {
|
||||
// supplied by a caller (claim, BeginRemediation, RecoverStale) can only write
|
||||
// the pairs asserted here. Pinning them to the shared table keeps a query
|
||||
// edit from moving a write off the table without this file failing.
|
||||
// The step edge RecoverStale writes is table-routed too: the sweep's WHERE
|
||||
// clause pins status = 'running' and writes 'interrupted', so the pair must
|
||||
// stay legal on the STEP table for a query edit to stay on it. The step table
|
||||
// is a different table from the execution's, so it is pinned beside it rather
|
||||
// than mixed into the execution pairs above.
|
||||
func TestSQLPinnedStepTransitionsAreTableLegal(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
from, to taskstate.StepStatus
|
||||
}{{name: "crash recovery pins step running->interrupted", from: taskstate.StepRunning, to: taskstate.StepInterrupted}} {
|
||||
if !taskstate.CanStepTransition(tc.from, tc.to) {
|
||||
t.Errorf("%s: %s -> %s is not in the shared step transition table", tc.name, tc.from, tc.to)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLPinnedTransitionsAreTableLegal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in new issue
Block a user