diff --git a/internal/domain/workflow/workflow.go b/internal/domain/workflow/workflow.go index 41e5594e0..dac2a7a71 100644 --- a/internal/domain/workflow/workflow.go +++ b/internal/domain/workflow/workflow.go @@ -2,6 +2,7 @@ package workflow import ( + "bytes" "context" "encoding/json" "fmt" @@ -374,7 +375,8 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { park(ctx, tc, err.Error()) return } - for _, stage := range stages { + from, history := tc.Task.ResumeFrom, tc.Task.ResumeHistory + for i, stage := range stages { tc.Stage = stage.Name if err := tc.Store.Update(ctx, t); err != nil { park(ctx, tc, fmt.Sprintf("persist task before stage %s: %v", stage.Name, err)) @@ -406,17 +408,7 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { // Shutdown is not a failure: record the step as interrupted and return // without parking. if ctx.Err() != nil && err != nil { - finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) - finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{ - StepID: stepID, ExecutionID: t.ID, - From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(), - }) - cancel() - if recordErr != nil { - stageLog.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage.Name, "err", recordErr) - } else { - publishEvent(ctx, tc, finishEvent) - } + recordInterrupted(ctx, tc, stepID, stage.Name, err, stageLog) stageLog.InfoContext(ctx, "stage interrupted", "err", err) return } @@ -435,6 +427,9 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { return } publishEvent(ctx, tc, finishEvent) + if i == 0 { + tc.consumeResume(from, history) + } if ended := endsRun(ctx, tc, stage, err, log, stageLog); ended { return @@ -445,6 +440,22 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) { finish(ctx, tc, log) } +// recordInterrupted records a stage that shutdown stopped, on a context the +// shutdown has not cancelled. +func recordInterrupted(ctx context.Context, tc *TaskContext, stepID int64, stage string, err error, log *slog.Logger) { + finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{ + StepID: stepID, ExecutionID: tc.Task.ID, + From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(), + }) + cancel() + if recordErr != nil { + log.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage, "err", recordErr) + } else { + publishEvent(ctx, tc, finishEvent) + } +} + // stageOutcome is how a stage that ran to its end records: a stage whose when // was false is skipped, with the condition as its detail; an error fails it; // anything else succeeds. Interruption is the caller's call, because it owns @@ -479,11 +490,9 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) { if from == "" { return stages, nil } - // The point is consumed: a run that stops again records its own, and a - // plain requeue after this one starts from the first step. - defer func() { tc.Task.ResumeFrom, tc.Task.ResumeResults, tc.Task.ResumeHistory = "", nil, nil }() at := slices.IndexFunc(stages, func(s Stage) bool { return s.Name == from }) if at < 0 { + tc.consumeResume(from, tc.Task.ResumeHistory) return nil, fmt.Errorf("resume point %q is not a step of this workflow", from) } if len(tc.Task.ResumeResults) > 0 { @@ -500,6 +509,21 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) { return stages[at:], nil } +// consumeResume clears the resume point. Run does it once the first resumed +// stage has finished, not when the point is read: until then the row must +// keep the point so a daemon crash inside that stage resumes there again, +// and a run that stops later records its own. +// A conversation the stage itself stopped with is a new one and stays. +func (tc *TaskContext) consumeResume(from string, history json.RawMessage) { + if from == "" { + return + } + tc.Task.ResumeFrom, tc.Task.ResumeResults = "", nil + if bytes.Equal(tc.Task.ResumeHistory, history) { + tc.Task.ResumeHistory = nil + } +} + // endsRun applies a recorded stage's result: a failure parks unless the stage // continues on failure, and an outcome finishes the run. func endsRun(ctx context.Context, tc *TaskContext, stage Stage, err error, log, stageLog *slog.Logger) bool { diff --git a/internal/infrastructure/postgres/migrations/0071_restart_requeues.sql b/internal/infrastructure/postgres/migrations/0071_restart_requeues.sql new file mode 100644 index 000000000..be2efe9b1 --- /dev/null +++ b/internal/infrastructure/postgres/migrations/0071_restart_requeues.sql @@ -0,0 +1,10 @@ +-- tasks.restart_requeues counts the daemon restarts that re-queued the task +-- while it ran; at the cap the task parks, and an operator retry resets it. + +-- +goose Up + +ALTER TABLE tasks ADD COLUMN restart_requeues bigint NOT NULL DEFAULT 0; + +-- +goose Down + +ALTER TABLE tasks DROP COLUMN restart_requeues; diff --git a/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go b/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go index 12a71a357..3c7f9845c 100644 --- a/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go +++ b/internal/infrastructure/postgres/postgresdb/call_tasks.sql.go @@ -24,7 +24,7 @@ func (q *Queries) CallStatusDetail(ctx context.Context, taskID int64) (string, e const enqueueCallTask = `-- name: EnqueueCallTask :one WITH caller AS ( - SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history FROM tasks t WHERE t.id = $1 FOR UPDATE + SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history, t.restart_requeues FROM tasks t WHERE t.id = $1 FOR UPDATE ), number AS ( INSERT INTO chat_issue_numbers (owner, repo, last_number) SELECT caller.owner, caller.repo, $9::bigint + 1 FROM caller @@ -43,7 +43,7 @@ SELECT FROM caller, number WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues ` type EnqueueCallTaskParams struct { @@ -140,6 +140,7 @@ func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } diff --git a/internal/infrastructure/postgres/postgresdb/models.go b/internal/infrastructure/postgres/postgresdb/models.go index 5742c2d9c..f960f09c7 100644 --- a/internal/infrastructure/postgres/postgresdb/models.go +++ b/internal/infrastructure/postgres/postgresdb/models.go @@ -587,6 +587,7 @@ type Task struct { HandoffFromTaskID pgtype.Int8 HandoffToTaskID pgtype.Int8 ResumeHistory []byte + RestartRequeues int64 } type ToolCall struct { diff --git a/internal/infrastructure/postgres/postgresdb/steps.sql.go b/internal/infrastructure/postgres/postgresdb/steps.sql.go index d6080588c..769692a61 100644 --- a/internal/infrastructure/postgres/postgresdb/steps.sql.go +++ b/internal/infrastructure/postgres/postgresdb/steps.sql.go @@ -121,7 +121,7 @@ const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many UPDATE step_executions SET status = 'interrupted', finished_at = now() WHERE execution_id = $1 AND attempt = $2 AND status = 'running' -RETURNING id, name +RETURNING id, name, depth, kind ` type InterruptAttemptStepsParams struct { @@ -130,8 +130,10 @@ type InterruptAttemptStepsParams struct { } type InterruptAttemptStepsRow struct { - ID int64 - Name string + ID int64 + Name string + Depth int32 + Kind string } // The crash-recovery step edge (docs/prds/execution-tree-state-machine.md): @@ -148,7 +150,12 @@ func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemp var items []InterruptAttemptStepsRow for rows.Next() { var i InterruptAttemptStepsRow - if err := rows.Scan(&i.ID, &i.Name); err != nil { + if err := rows.Scan( + &i.ID, + &i.Name, + &i.Depth, + &i.Kind, + ); err != nil { return nil, err } items = append(items, i) @@ -299,19 +306,24 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu } const lockRunningExecutions = `-- name: LockRunningExecutions :many -SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt +SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt, + restart_requeues, retry_mode, resume_from, resume_results FROM tasks WHERE status = 'running' FOR UPDATE ` type LockRunningExecutionsRow struct { - ID int64 - OrgID string - WorkspaceID string - Owner string - Repo string - IssueNumber int64 - Workflow string - Attempt int64 + ID int64 + OrgID string + WorkspaceID string + Owner string + Repo string + IssueNumber int64 + Workflow string + Attempt int64 + RestartRequeues int64 + RetryMode string + ResumeFrom string + ResumeResults []byte } // The executions a crashed or replaced daemon left running, locked for the @@ -335,6 +347,10 @@ func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecu &i.IssueNumber, &i.Workflow, &i.Attempt, + &i.RestartRequeues, + &i.RetryMode, + &i.ResumeFrom, + &i.ResumeResults, ); err != nil { return nil, err } @@ -395,6 +411,29 @@ func (q *Queries) LockStepExecution(ctx context.Context, id int64) (LockStepExec return i, err } +const priorStageResults = `-- name: PriorStageResults :one +SELECT results FROM step_executions +WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage' + AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL +ORDER BY id DESC +LIMIT 1 +` + +type PriorStageResultsParams struct { + ExecutionID int64 + Attempt int64 + ID int64 +} + +// What the attempt's last finished root stage before the given step recorded, +// the results a restart resumes that step with. +func (q *Queries) PriorStageResults(ctx context.Context, arg PriorStageResultsParams) ([]byte, error) { + row := q.db.QueryRow(ctx, priorStageResults, arg.ExecutionID, arg.Attempt, arg.ID) + var results []byte + err := row.Scan(&results) + return results, err +} + const startStepExecution = `-- name: StartStepExecution :execrows UPDATE step_executions SET status = 'running', started_at = now() WHERE id = $1 AND status = 'pending' diff --git a/internal/infrastructure/postgres/postgresdb/tasks.sql.go b/internal/infrastructure/postgres/postgresdb/tasks.sql.go index d2e9dbc02..9e0106a59 100644 --- a/internal/infrastructure/postgres/postgresdb/tasks.sql.go +++ b/internal/infrastructure/postgres/postgresdb/tasks.sql.go @@ -13,7 +13,7 @@ import ( ) const activeTasksByOrigin = `-- name: ActiveTasksByOrigin :many -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE origin = $1 AND origin <> '' AND status IN ('queued', 'running') AND ($2::text IS NULL OR org_id = $2) ORDER BY id ` @@ -88,6 +88,7 @@ func (q *Queries) ActiveTasksByOrigin(ctx context.Context, arg ActiveTasksByOrig &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ); err != nil { return nil, err } @@ -167,7 +168,7 @@ func (q *Queries) BeginRemediationTask(ctx context.Context, arg BeginRemediation const claimByIssue = `-- name: ClaimByIssue :one UPDATE tasks SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now() WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND status = 'queued' -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues ` type ClaimByIssueParams struct { @@ -236,6 +237,7 @@ func (q *Queries) ClaimByIssue(ctx context.Context, arg ClaimByIssueParams) (Tas &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } @@ -250,7 +252,7 @@ WHERE id = ( FOR UPDATE SKIP LOCKED LIMIT 1 ) -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues ` // FOR UPDATE SKIP LOCKED replaces the SQLite single-writer assumption: two @@ -317,6 +319,7 @@ func (q *Queries) ClaimNextTask(ctx context.Context) (Task, error) { &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } @@ -436,7 +439,7 @@ FROM number, (SELECT COALESCE( 'org-sys' ) AS id) AS owner_org WHERE $11::text IS NULL OR owner_org.id = $11 -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues ` type InsertChatTaskParams struct { @@ -537,6 +540,7 @@ func (q *Queries) InsertChatTask(ctx context.Context, arg InsertChatTaskParams) &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } @@ -549,7 +553,7 @@ SELECT $1, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, $2, t.sou t.org_id, t.workspace_id, t.inputs, t.plan, t.plan_document, t.binding_id, t.binding_version, t.binding_signature, t.binding_addr, t.origin, t.call_parent_task_id, t.call_depth, t.id FROM tasks t WHERE t.id = $3 -RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history +RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues ` type InsertHandoffTaskParams struct { @@ -618,6 +622,7 @@ func (q *Queries) InsertHandoffTask(ctx context.Context, arg InsertHandoffTaskPa &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } @@ -908,6 +913,24 @@ func (q *Queries) NextTaskID(ctx context.Context) (int64, error) { return column_1, err } +const parkRecoveredTask = `-- name: ParkRecoveredTask :execrows +UPDATE tasks SET status = 'parked', park_reason = $1, park_class = 'needs_decision', updated_at = now() +WHERE id = $2 AND status = 'running' +` + +type ParkRecoveredTaskParams struct { + ParkReason string + ID int64 +} + +func (q *Queries) ParkRecoveredTask(ctx context.Context, arg ParkRecoveredTaskParams) (int64, error) { + result, err := q.db.Exec(ctx, parkRecoveredTask, arg.ParkReason, arg.ID) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + const parkTask = `-- name: ParkTask :execrows UPDATE tasks SET status = 'parked', park_reason = $2, park_class = $3, park_blocker = $5, updated_at = now() @@ -957,12 +980,31 @@ func (q *Queries) PromotePendingReview(ctx context.Context, id int64) (int64, er return result.RowsAffected(), nil } -const recoverStaleTasks = `-- name: RecoverStaleTasks :execrows -UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running' +const requeueRecoveredTask = `-- name: RequeueRecoveredTask :execrows +UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1, + retry_mode = $1::text, resume_results = $2::jsonb, + resume_history = CASE WHEN resume_from = $3::text THEN resume_history ELSE 'null' END, + resume_from = $3::text, updated_at = now() +WHERE id = $4 AND status = 'running' ` -func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) { - result, err := q.db.Exec(ctx, recoverStaleTasks) +type RequeueRecoveredTaskParams struct { + RetryMode string + ResumeResults []byte + ResumeFrom string + ID int64 +} + +// The restart's re-queue of a task it found running, resuming where the +// interrupted attempt stood. The conversation is kept only for the step the +// row already resumed at. +func (q *Queries) RequeueRecoveredTask(ctx context.Context, arg RequeueRecoveredTaskParams) (int64, error) { + result, err := q.db.Exec(ctx, requeueRecoveredTask, + arg.RetryMode, + arg.ResumeResults, + arg.ResumeFrom, + arg.ID, + ) if err != nil { return 0, err } @@ -972,7 +1014,7 @@ func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) { const requeueTask = `-- name: RequeueTask :execrows UPDATE tasks SET status = 'queued', workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END, - park_reason = '', park_class = 'needs_decision', updated_at = now() + park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now() WHERE id = $2 AND status = $3 ` @@ -1042,7 +1084,7 @@ func (q *Queries) RespondReviewGateTask(ctx context.Context, arg RespondReviewGa } const retryTask = `-- name: RetryTask :execrows -UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, +UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0, workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END, retry_mode = $2::text, resume_from = $3::text, resume_results = $4::jsonb, resume_history = 'null', @@ -1142,7 +1184,7 @@ func (q *Queries) StampTaskBinding(ctx context.Context, arg StampTaskBindingPara } const taskByID = `-- name: TaskByID :one -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2) +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2) ` type TaskByIDParams struct { @@ -1208,12 +1250,13 @@ func (q *Queries) TaskByID(ctx context.Context, arg TaskByIDParams) (Task, error &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } const taskByIssue = `-- name: TaskByIssue :one -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL AND ($4::text IS NULL OR org_id = $4) ` @@ -1287,12 +1330,13 @@ func (q *Queries) TaskByIssue(ctx context.Context, arg TaskByIssueParams) (Task, &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } const taskByPR = `-- name: TaskByPR :one -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4 +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4 AND ($5::text IS NULL OR org_id = $5) ` @@ -1371,6 +1415,7 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ) return i, err } @@ -1423,7 +1468,7 @@ func (q *Queries) TransitionTask(ctx context.Context, arg TransitionTaskParams) } const unpostedOutcomes = `-- name: UnpostedOutcomes :many -SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks +SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE origin <> '' AND outcome_posted_status <> status AND status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed', 'waiting_approval', 'pr_open') AND handoff_to_task_id IS NULL @@ -1498,6 +1543,7 @@ func (q *Queries) UnpostedOutcomes(ctx context.Context, scopeOrg pgtype.Text) ([ &i.HandoffFromTaskID, &i.HandoffToTaskID, &i.ResumeHistory, + &i.RestartRequeues, ); err != nil { return nil, err } diff --git a/internal/infrastructure/postgres/queries/steps.sql b/internal/infrastructure/postgres/queries/steps.sql index d061bc0a1..a896b8d6c 100644 --- a/internal/infrastructure/postgres/queries/steps.sql +++ b/internal/infrastructure/postgres/queries/steps.sql @@ -56,7 +56,8 @@ RETURNING id, name; -- The executions a crashed or replaced daemon left running, locked for the -- recovery transaction so the step sweep and the requeue cannot race a -- concurrent write. The row carries the identity the steps' events need. -SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt +SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt, + restart_requeues, retry_mode, resume_from, resume_results FROM tasks WHERE status = 'running' FOR UPDATE; -- name: InterruptAttemptSteps :many @@ -68,7 +69,7 @@ FROM tasks WHERE status = 'running' FOR UPDATE; UPDATE step_executions SET status = 'interrupted', finished_at = now() WHERE execution_id = $1 AND attempt = $2 AND status = 'running' -RETURNING id, name; +RETURNING id, name, depth, kind; -- name: ListStepExecutions :many -- attempt = 0 lists every attempt of the execution, oldest first; the index @@ -89,6 +90,15 @@ SELECT id, called_execution_id FROM step_executions WHERE execution_id = $1 AND attempt = $2 AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0; +-- name: PriorStageResults :one +-- What the attempt's last finished root stage before the given step recorded, +-- the results a restart resumes that step with. +SELECT results FROM step_executions +WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage' + AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL +ORDER BY id DESC +LIMIT 1; + -- name: LatestStageResults :one -- The most recent finished run of a root stage, across attempts: a resumed -- attempt skips the stages before its resume point, so the last attempt that diff --git a/internal/infrastructure/postgres/queries/tasks.sql b/internal/infrastructure/postgres/queries/tasks.sql index bb85c1768..e6e3ccc5e 100644 --- a/internal/infrastructure/postgres/queries/tasks.sql +++ b/internal/infrastructure/postgres/queries/tasks.sql @@ -188,8 +188,19 @@ DELETE FROM tasks WHERE status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed') AND (sqlc.narg(scope_org)::text IS NULL OR org_id = sqlc.narg(scope_org)); --- name: RecoverStaleTasks :execrows -UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running'; +-- name: RequeueRecoveredTask :execrows +-- The restart's re-queue of a task it found running, resuming where the +-- interrupted attempt stood. The conversation is kept only for the step the +-- row already resumed at. +UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1, + retry_mode = @retry_mode::text, resume_results = @resume_results::jsonb, + resume_history = CASE WHEN resume_from = @resume_from::text THEN resume_history ELSE 'null' END, + resume_from = @resume_from::text, updated_at = now() +WHERE id = @id AND status = 'running'; + +-- name: ParkRecoveredTask :execrows +UPDATE tasks SET status = 'parked', park_reason = @park_reason, park_class = 'needs_decision', updated_at = now() +WHERE id = @id AND status = 'running'; -- name: EnqueueIssue :execrows -- The org is derived from the identity at the one place a task row is @@ -265,11 +276,11 @@ WHERE id = $1 AND status = 'pr_open'; -- name: RequeueTask :execrows UPDATE tasks SET status = 'queued', workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END, - park_reason = '', park_class = 'needs_decision', updated_at = now() + park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now() WHERE id = @id AND status = @from_status; -- name: RetryTask :execrows -UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, +UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0, workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END, retry_mode = @retry_mode::text, resume_from = @resume_from::text, resume_results = @resume_results::jsonb, resume_history = 'null', diff --git a/internal/infrastructure/postgres/store.go b/internal/infrastructure/postgres/store.go index ae308d223..60ee5ce07 100644 --- a/internal/infrastructure/postgres/store.go +++ b/internal/infrastructure/postgres/store.go @@ -702,8 +702,9 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string return eventID, nil } -// RecoverStale requeues tasks left running and marks their running steps -// interrupted, in one transaction. +// RecoverStale requeues tasks left running, resuming each at the step the +// restart interrupted, and marks their running steps interrupted, in one +// transaction. A task already requeued too often parks instead. func (s *Store) RecoverStale(ctx context.Context) (int64, error) { tx, err := s.pool.Begin(ctx) if err != nil { @@ -723,28 +724,60 @@ func (s *Store) RecoverStale(ctx context.Context) (int64, error) { if err != nil { return 0, err } + in := taskstate.RecoveryInput{ + Restarts: int(execution.RestartRequeues), RetryMode: execution.RetryMode, + ResumeFrom: execution.ResumeFrom, ResumeResults: execution.ResumeResults, + } for _, step := range steps { if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID, execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow, int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil { return 0, err } + if step.Depth != 0 || step.Kind != task.StepKindStage { + continue + } + in.Stage = step.Name + prior, err := q.PriorStageResults(ctx, postgresdb.PriorStageResultsParams{ + ExecutionID: execution.ID, Attempt: execution.Attempt, ID: step.ID, + }) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return 0, err + } + in.PriorResults = prior } - if err := recordTransition(ctx, q, postgresdb.InsertTransitionParams{ - TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued, - Detail: "re-queued after a crashed or replaced daemon", - }); err != nil { + if err := recoverTask(ctx, q, execution.ID, taskstate.PlanRecovery(in)); err != nil { return 0, err } } - n, err := q.RecoverStaleTasks(ctx) - if err != nil { - return 0, err - } if err := tx.Commit(ctx); err != nil { return 0, err } - return n, nil + return int64(len(executions)), nil +} + +// recoverTask writes one restart-found task's recovery and its transition. +func recoverTask(ctx context.Context, q *postgresdb.Queries, id int64, rec taskstate.Recovery) error { + to, detail := workflow.StatusQueued, "re-queued after a crashed or replaced daemon" + var n int64 + var err error + if rec.Park { + to, detail = workflow.StatusParked, rec.ParkReason + n, err = q.ParkRecoveredTask(ctx, postgresdb.ParkRecoveredTaskParams{ID: id, ParkReason: rec.ParkReason}) + } else { + n, err = q.RequeueRecoveredTask(ctx, postgresdb.RequeueRecoveredTaskParams{ + ID: id, RetryMode: rec.RetryMode, ResumeFrom: rec.ResumeFrom, ResumeResults: rec.ResumeResults, + }) + } + if err != nil { + return err + } + if n == 0 { + return storecontract.ErrStaleTransition + } + return recordTransition(ctx, q, postgresdb.InsertTransitionParams{ + TaskID: id, FromStatus: workflow.StatusRunning, ToStatus: to, Detail: detail, + }) } // OpenPRs returns tasks whose PR state should be reconciled with the forge, diff --git a/internal/taskstate/recovery_test.go b/internal/taskstate/recovery_test.go new file mode 100644 index 000000000..d8885d3fa --- /dev/null +++ b/internal/taskstate/recovery_test.go @@ -0,0 +1,48 @@ +package taskstate + +import ( + "testing" +) + +func TestPlanRecovery(t *testing.T) { + tests := []struct { + name string + give RecoveryInput + want Recovery + }{ + { + name: "resumes at the interrupted step with the finished steps' results", + give: RecoveryInput{Stage: "build", PriorResults: []byte(`{"plan":1}`)}, + want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)}, + }, + { + name: "a crash inside the first resumed step keeps the row's resume point", + give: RecoveryInput{Stage: "build", ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)}, + want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)}, + }, + { + name: "the first step of a fresh attempt starts over", + give: RecoveryInput{Stage: "plan", RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)}, + want: Recovery{RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)}, + }, + { + name: "no running step leaves the row as it is", + give: RecoveryInput{ResumeResults: []byte(`{}`)}, + want: Recovery{ResumeResults: []byte(`{}`)}, + }, + { + name: "parks once the cap is spent", + give: RecoveryInput{Restarts: MaxRestartRequeues, Stage: "build", PriorResults: []byte(`{}`)}, + want: Recovery{Park: true, ParkReason: "interrupted by 3 daemon restarts in a row; retry to run it again"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := PlanRecovery(tt.give) + if got.Park != tt.want.Park || got.ParkReason != tt.want.ParkReason || got.RetryMode != tt.want.RetryMode || + got.ResumeFrom != tt.want.ResumeFrom || string(got.ResumeResults) != string(tt.want.ResumeResults) { + t.Errorf("PlanRecovery() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/internal/taskstate/taskstate.go b/internal/taskstate/taskstate.go index eb584ab3b..dc3b5ccaa 100644 --- a/internal/taskstate/taskstate.go +++ b/internal/taskstate/taskstate.go @@ -356,3 +356,56 @@ func TransientRetryDue(class string, retries int, parkedAt, now time.Time) bool } return !now.Before(parkedAt.Add(transientBackoffBase << retries)) } + +// MaxRestartRequeues is how many times daemon restarts may re-queue a task +// that was running before it parks for an operator. +const MaxRestartRequeues = 3 + +// RecoveryInput is what the store knows about a task a restart found running. +type RecoveryInput struct { + // Restarts is how many earlier restarts re-queued the task. + Restarts int + // Stage is the root step the restart interrupted, "" when none was running. + Stage string + // PriorResults are the step results recorded by the last step this attempt + // finished before Stage, nil when it finished none. + PriorResults []byte + // RetryMode, ResumeFrom and ResumeResults are the task row's own. + RetryMode, ResumeFrom string + ResumeResults []byte +} + +// Recovery is how a task a restart found running continues. +type Recovery struct { + // Park reports that the task goes to an operator instead of the queue. + Park bool + ParkReason string + // RetryMode, ResumeFrom and ResumeResults are written to the re-queued row. + RetryMode, ResumeFrom string + ResumeResults []byte +} + +// PlanRecovery decides how a restart treats a task left running: it resumes +// at the interrupted step in its kept worktree, with the results of the steps +// before it, rather than repaying them. A task already re-queued +// MaxRestartRequeues times parks instead. +func PlanRecovery(in RecoveryInput) Recovery { + if in.Restarts >= MaxRestartRequeues { + return Recovery{Park: true, ParkReason: fmt.Sprintf("interrupted by %d daemon restarts in a row; retry to run it again", in.Restarts)} + } + rec := Recovery{RetryMode: in.RetryMode, ResumeFrom: in.ResumeFrom, ResumeResults: in.ResumeResults} + if in.Stage == "" { + return rec + } + switch { + case in.PriorResults != nil: + rec.ResumeFrom, rec.ResumeResults = in.Stage, in.PriorResults + case in.ResumeFrom == in.Stage: + // The attempt was itself resumed at this step and finished none since. + default: + // The first step of a fresh attempt: nothing is finished, so start over. + return Recovery{RetryMode: in.RetryMode, ResumeResults: []byte("{}")} + } + rec.RetryMode = string(RetryKeepWorktree) + return rec +}