fix(store): a restart re-queues a running task at its interrupted step and parks it after 3 restarts (#1841)
deploy / plan (push) Successful in 7s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s
deploy / gate (push) Canceled after 4m4s

This commit is contained in:
sam committed 2026-11-09 14:36:08 +11:00
1 parent f44db54948
commit e75d5c8ade
11 files changed
+338 -62

No files matched your search

+39 -15
View File
@@ -2,6 +2,7 @@
package workflow
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -374,7 +375,8 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
park(ctx, tc, err.Error())
return
}
for _, stage := range stages {
from, history := tc.Task.ResumeFrom, tc.Task.ResumeHistory
for i, stage := range stages {
tc.Stage = stage.Name
if err := tc.Store.Update(ctx, t); err != nil {
park(ctx, tc, fmt.Sprintf("persist task before stage %s: %v", stage.Name, err))
@@ -406,17 +408,7 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
// Shutdown is not a failure: record the step as interrupted and return
// without parking.
if ctx.Err() != nil && err != nil {
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{
StepID: stepID, ExecutionID: t.ID,
From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(),
})
cancel()
if recordErr != nil {
stageLog.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage.Name, "err", recordErr)
} else {
publishEvent(ctx, tc, finishEvent)
}
recordInterrupted(ctx, tc, stepID, stage.Name, err, stageLog)
stageLog.InfoContext(ctx, "stage interrupted", "err", err)
return
}
@@ -435,6 +427,9 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
return
}
publishEvent(ctx, tc, finishEvent)
if i == 0 {
tc.consumeResume(from, history)
}
if ended := endsRun(ctx, tc, stage, err, log, stageLog); ended {
return
@@ -445,6 +440,22 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
finish(ctx, tc, log)
}
// recordInterrupted records a stage that shutdown stopped, on a context the
// shutdown has not cancelled.
func recordInterrupted(ctx context.Context, tc *TaskContext, stepID int64, stage string, err error, log *slog.Logger) {
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{
StepID: stepID, ExecutionID: tc.Task.ID,
From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(),
})
cancel()
if recordErr != nil {
log.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage, "err", recordErr)
} else {
publishEvent(ctx, tc, finishEvent)
}
}
// stageOutcome is how a stage that ran to its end records: a stage whose when
// was false is skipped, with the condition as its detail; an error fails it;
// anything else succeeds. Interruption is the caller's call, because it owns
@@ -479,11 +490,9 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
if from == "" {
return stages, nil
}
// The point is consumed: a run that stops again records its own, and a
// plain requeue after this one starts from the first step.
defer func() { tc.Task.ResumeFrom, tc.Task.ResumeResults, tc.Task.ResumeHistory = "", nil, nil }()
at := slices.IndexFunc(stages, func(s Stage) bool { return s.Name == from })
if at < 0 {
tc.consumeResume(from, tc.Task.ResumeHistory)
return nil, fmt.Errorf("resume point %q is not a step of this workflow", from)
}
if len(tc.Task.ResumeResults) > 0 {
@@ -500,6 +509,21 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
return stages[at:], nil
}
// consumeResume clears the resume point. Run does it once the first resumed
// stage has finished, not when the point is read: until then the row must
// keep the point so a daemon crash inside that stage resumes there again,
// and a run that stops later records its own.
// A conversation the stage itself stopped with is a new one and stays.
func (tc *TaskContext) consumeResume(from string, history json.RawMessage) {
if from == "" {
return
}
tc.Task.ResumeFrom, tc.Task.ResumeResults = "", nil
if bytes.Equal(tc.Task.ResumeHistory, history) {
tc.Task.ResumeHistory = nil
}
}
// endsRun applies a recorded stage's result: a failure parks unless the stage
// continues on failure, and an outcome finishes the run.
func endsRun(ctx context.Context, tc *TaskContext, stage Stage, err error, log, stageLog *slog.Logger) bool {
@@ -0,0 +1,10 @@
-- tasks.restart_requeues counts the daemon restarts that re-queued the task
-- while it ran; at the cap the task parks, and an operator retry resets it.
-- +goose Up
ALTER TABLE tasks ADD COLUMN restart_requeues bigint NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE tasks DROP COLUMN restart_requeues;
@@ -24,7 +24,7 @@ func (q *Queries) CallStatusDetail(ctx context.Context, taskID int64) (string, e
const enqueueCallTask = `-- name: EnqueueCallTask :one
WITH caller AS (
SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history FROM tasks t WHERE t.id = $1 FOR UPDATE
SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history, t.restart_requeues FROM tasks t WHERE t.id = $1 FOR UPDATE
), number AS (
INSERT INTO chat_issue_numbers (owner, repo, last_number)
SELECT caller.owner, caller.repo, $9::bigint + 1 FROM caller
@@ -43,7 +43,7 @@ SELECT
FROM caller, number
WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
`
type EnqueueCallTaskParams struct {
@@ -140,6 +140,7 @@ func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
@@ -587,6 +587,7 @@ type Task struct {
HandoffFromTaskID pgtype.Int8
HandoffToTaskID pgtype.Int8
ResumeHistory []byte
RestartRequeues int64
}
type ToolCall struct {
@@ -121,7 +121,7 @@ const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many
UPDATE step_executions
SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name
RETURNING id, name, depth, kind
`
type InterruptAttemptStepsParams struct {
@@ -130,8 +130,10 @@ type InterruptAttemptStepsParams struct {
}
type InterruptAttemptStepsRow struct {
ID int64
Name string
ID int64
Name string
Depth int32
Kind string
}
// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
@@ -148,7 +150,12 @@ func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemp
var items []InterruptAttemptStepsRow
for rows.Next() {
var i InterruptAttemptStepsRow
if err := rows.Scan(&i.ID, &i.Name); err != nil {
if err := rows.Scan(
&i.ID,
&i.Name,
&i.Depth,
&i.Kind,
); err != nil {
return nil, err
}
items = append(items, i)
@@ -299,19 +306,24 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu
}
const lockRunningExecutions = `-- name: LockRunningExecutions :many
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt,
restart_requeues, retry_mode, resume_from, resume_results
FROM tasks WHERE status = 'running' FOR UPDATE
`
type LockRunningExecutionsRow struct {
ID int64
OrgID string
WorkspaceID string
Owner string
Repo string
IssueNumber int64
Workflow string
Attempt int64
ID int64
OrgID string
WorkspaceID string
Owner string
Repo string
IssueNumber int64
Workflow string
Attempt int64
RestartRequeues int64
RetryMode string
ResumeFrom string
ResumeResults []byte
}
// The executions a crashed or replaced daemon left running, locked for the
@@ -335,6 +347,10 @@ func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecu
&i.IssueNumber,
&i.Workflow,
&i.Attempt,
&i.RestartRequeues,
&i.RetryMode,
&i.ResumeFrom,
&i.ResumeResults,
); err != nil {
return nil, err
}
@@ -395,6 +411,29 @@ func (q *Queries) LockStepExecution(ctx context.Context, id int64) (LockStepExec
return i, err
}
const priorStageResults = `-- name: PriorStageResults :one
SELECT results FROM step_executions
WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage'
AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL
ORDER BY id DESC
LIMIT 1
`
type PriorStageResultsParams struct {
ExecutionID int64
Attempt int64
ID int64
}
// What the attempt's last finished root stage before the given step recorded,
// the results a restart resumes that step with.
func (q *Queries) PriorStageResults(ctx context.Context, arg PriorStageResultsParams) ([]byte, error) {
row := q.db.QueryRow(ctx, priorStageResults, arg.ExecutionID, arg.Attempt, arg.ID)
var results []byte
err := row.Scan(&results)
return results, err
}
const startStepExecution = `-- name: StartStepExecution :execrows
UPDATE step_executions SET status = 'running', started_at = now()
WHERE id = $1 AND status = 'pending'
@@ -13,7 +13,7 @@ import (
)
const activeTasksByOrigin = `-- name: ActiveTasksByOrigin :many
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks
WHERE origin = $1 AND origin <> '' AND status IN ('queued', 'running') AND ($2::text IS NULL OR org_id = $2)
ORDER BY id
`
@@ -88,6 +88,7 @@ func (q *Queries) ActiveTasksByOrigin(ctx context.Context, arg ActiveTasksByOrig
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
); err != nil {
return nil, err
}
@@ -167,7 +168,7 @@ func (q *Queries) BeginRemediationTask(ctx context.Context, arg BeginRemediation
const claimByIssue = `-- name: ClaimByIssue :one
UPDATE tasks SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now()
WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND status = 'queued'
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
`
type ClaimByIssueParams struct {
@@ -236,6 +237,7 @@ func (q *Queries) ClaimByIssue(ctx context.Context, arg ClaimByIssueParams) (Tas
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
@@ -250,7 +252,7 @@ WHERE id = (
FOR UPDATE SKIP LOCKED
LIMIT 1
)
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
`
// FOR UPDATE SKIP LOCKED replaces the SQLite single-writer assumption: two
@@ -317,6 +319,7 @@ func (q *Queries) ClaimNextTask(ctx context.Context) (Task, error) {
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
@@ -436,7 +439,7 @@ FROM number, (SELECT COALESCE(
'org-sys'
) AS id) AS owner_org
WHERE $11::text IS NULL OR owner_org.id = $11
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
`
type InsertChatTaskParams struct {
@@ -537,6 +540,7 @@ func (q *Queries) InsertChatTask(ctx context.Context, arg InsertChatTaskParams)
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
@@ -549,7 +553,7 @@ SELECT $1, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, $2, t.sou
t.org_id, t.workspace_id, t.inputs, t.plan, t.plan_document, t.binding_id, t.binding_version, t.binding_signature,
t.binding_addr, t.origin, t.call_parent_task_id, t.call_depth, t.id
FROM tasks t WHERE t.id = $3
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
`
type InsertHandoffTaskParams struct {
@@ -618,6 +622,7 @@ func (q *Queries) InsertHandoffTask(ctx context.Context, arg InsertHandoffTaskPa
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
@@ -908,6 +913,24 @@ func (q *Queries) NextTaskID(ctx context.Context) (int64, error) {
return column_1, err
}
const parkRecoveredTask = `-- name: ParkRecoveredTask :execrows
UPDATE tasks SET status = 'parked', park_reason = $1, park_class = 'needs_decision', updated_at = now()
WHERE id = $2 AND status = 'running'
`
type ParkRecoveredTaskParams struct {
ParkReason string
ID int64
}
func (q *Queries) ParkRecoveredTask(ctx context.Context, arg ParkRecoveredTaskParams) (int64, error) {
result, err := q.db.Exec(ctx, parkRecoveredTask, arg.ParkReason, arg.ID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const parkTask = `-- name: ParkTask :execrows
UPDATE tasks
SET status = 'parked', park_reason = $2, park_class = $3, park_blocker = $5, updated_at = now()
@@ -957,12 +980,31 @@ func (q *Queries) PromotePendingReview(ctx context.Context, id int64) (int64, er
return result.RowsAffected(), nil
}
const recoverStaleTasks = `-- name: RecoverStaleTasks :execrows
UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running'
const requeueRecoveredTask = `-- name: RequeueRecoveredTask :execrows
UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1,
retry_mode = $1::text, resume_results = $2::jsonb,
resume_history = CASE WHEN resume_from = $3::text THEN resume_history ELSE 'null' END,
resume_from = $3::text, updated_at = now()
WHERE id = $4 AND status = 'running'
`
func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) {
result, err := q.db.Exec(ctx, recoverStaleTasks)
type RequeueRecoveredTaskParams struct {
RetryMode string
ResumeResults []byte
ResumeFrom string
ID int64
}
// The restart's re-queue of a task it found running, resuming where the
// interrupted attempt stood. The conversation is kept only for the step the
// row already resumed at.
func (q *Queries) RequeueRecoveredTask(ctx context.Context, arg RequeueRecoveredTaskParams) (int64, error) {
result, err := q.db.Exec(ctx, requeueRecoveredTask,
arg.RetryMode,
arg.ResumeResults,
arg.ResumeFrom,
arg.ID,
)
if err != nil {
return 0, err
}
@@ -972,7 +1014,7 @@ func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) {
const requeueTask = `-- name: RequeueTask :execrows
UPDATE tasks SET status = 'queued',
workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END,
park_reason = '', park_class = 'needs_decision', updated_at = now()
park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now()
WHERE id = $2 AND status = $3
`
@@ -1042,7 +1084,7 @@ func (q *Queries) RespondReviewGateTask(ctx context.Context, arg RespondReviewGa
}
const retryTask = `-- name: RetryTask :execrows
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1,
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0,
workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END,
retry_mode = $2::text,
resume_from = $3::text, resume_results = $4::jsonb, resume_history = 'null',
@@ -1142,7 +1184,7 @@ func (q *Queries) StampTaskBinding(ctx context.Context, arg StampTaskBindingPara
}
const taskByID = `-- name: TaskByID :one
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2)
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2)
`
type TaskByIDParams struct {
@@ -1208,12 +1250,13 @@ func (q *Queries) TaskByID(ctx context.Context, arg TaskByIDParams) (Task, error
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
const taskByIssue = `-- name: TaskByIssue :one
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL
AND ($4::text IS NULL OR org_id = $4)
`
@@ -1287,12 +1330,13 @@ func (q *Queries) TaskByIssue(ctx context.Context, arg TaskByIssueParams) (Task,
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
const taskByPR = `-- name: TaskByPR :one
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4
AND ($5::text IS NULL OR org_id = $5)
`
@@ -1371,6 +1415,7 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
)
return i, err
}
@@ -1423,7 +1468,7 @@ func (q *Queries) TransitionTask(ctx context.Context, arg TransitionTaskParams)
}
const unpostedOutcomes = `-- name: UnpostedOutcomes :many
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks
WHERE origin <> '' AND outcome_posted_status <> status
AND status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed', 'waiting_approval', 'pr_open')
AND handoff_to_task_id IS NULL
@@ -1498,6 +1543,7 @@ func (q *Queries) UnpostedOutcomes(ctx context.Context, scopeOrg pgtype.Text) ([
&i.HandoffFromTaskID,
&i.HandoffToTaskID,
&i.ResumeHistory,
&i.RestartRequeues,
); err != nil {
return nil, err
}
@@ -56,7 +56,8 @@ RETURNING id, name;
-- The executions a crashed or replaced daemon left running, locked for the
-- recovery transaction so the step sweep and the requeue cannot race a
-- concurrent write. The row carries the identity the steps' events need.
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt,
restart_requeues, retry_mode, resume_from, resume_results
FROM tasks WHERE status = 'running' FOR UPDATE;
-- name: InterruptAttemptSteps :many
@@ -68,7 +69,7 @@ FROM tasks WHERE status = 'running' FOR UPDATE;
UPDATE step_executions
SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name;
RETURNING id, name, depth, kind;
-- name: ListStepExecutions :many
-- attempt = 0 lists every attempt of the execution, oldest first; the index
@@ -89,6 +90,15 @@ SELECT id, called_execution_id FROM step_executions
WHERE execution_id = $1 AND attempt = $2
AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0;
-- name: PriorStageResults :one
-- What the attempt's last finished root stage before the given step recorded,
-- the results a restart resumes that step with.
SELECT results FROM step_executions
WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage'
AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL
ORDER BY id DESC
LIMIT 1;
-- name: LatestStageResults :one
-- The most recent finished run of a root stage, across attempts: a resumed
-- attempt skips the stages before its resume point, so the last attempt that
@@ -188,8 +188,19 @@ DELETE FROM tasks
WHERE status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed')
AND (sqlc.narg(scope_org)::text IS NULL OR org_id = sqlc.narg(scope_org));
-- name: RecoverStaleTasks :execrows
UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running';
-- name: RequeueRecoveredTask :execrows
-- The restart's re-queue of a task it found running, resuming where the
-- interrupted attempt stood. The conversation is kept only for the step the
-- row already resumed at.
UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1,
retry_mode = @retry_mode::text, resume_results = @resume_results::jsonb,
resume_history = CASE WHEN resume_from = @resume_from::text THEN resume_history ELSE 'null' END,
resume_from = @resume_from::text, updated_at = now()
WHERE id = @id AND status = 'running';
-- name: ParkRecoveredTask :execrows
UPDATE tasks SET status = 'parked', park_reason = @park_reason, park_class = 'needs_decision', updated_at = now()
WHERE id = @id AND status = 'running';
-- name: EnqueueIssue :execrows
-- The org is derived from the identity at the one place a task row is
@@ -265,11 +276,11 @@ WHERE id = $1 AND status = 'pr_open';
-- name: RequeueTask :execrows
UPDATE tasks SET status = 'queued',
workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END,
park_reason = '', park_class = 'needs_decision', updated_at = now()
park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now()
WHERE id = @id AND status = @from_status;
-- name: RetryTask :execrows
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1,
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0,
workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END,
retry_mode = @retry_mode::text,
resume_from = @resume_from::text, resume_results = @resume_results::jsonb, resume_history = 'null',
+44 -11
View File
@@ -702,8 +702,9 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
return eventID, nil
}
// RecoverStale requeues tasks left running and marks their running steps
// interrupted, in one transaction.
// RecoverStale requeues tasks left running, resuming each at the step the
// restart interrupted, and marks their running steps interrupted, in one
// transaction. A task already requeued too often parks instead.
func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
@@ -723,28 +724,60 @@ func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
if err != nil {
return 0, err
}
in := taskstate.RecoveryInput{
Restarts: int(execution.RestartRequeues), RetryMode: execution.RetryMode,
ResumeFrom: execution.ResumeFrom, ResumeResults: execution.ResumeResults,
}
for _, step := range steps {
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil {
return 0, err
}
if step.Depth != 0 || step.Kind != task.StepKindStage {
continue
}
in.Stage = step.Name
prior, err := q.PriorStageResults(ctx, postgresdb.PriorStageResultsParams{
ExecutionID: execution.ID, Attempt: execution.Attempt, ID: step.ID,
})
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return 0, err
}
in.PriorResults = prior
}
if err := recordTransition(ctx, q, postgresdb.InsertTransitionParams{
TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued,
Detail: "re-queued after a crashed or replaced daemon",
}); err != nil {
if err := recoverTask(ctx, q, execution.ID, taskstate.PlanRecovery(in)); err != nil {
return 0, err
}
}
n, err := q.RecoverStaleTasks(ctx)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, err
}
return n, nil
return int64(len(executions)), nil
}
// recoverTask writes one restart-found task's recovery and its transition.
func recoverTask(ctx context.Context, q *postgresdb.Queries, id int64, rec taskstate.Recovery) error {
to, detail := workflow.StatusQueued, "re-queued after a crashed or replaced daemon"
var n int64
var err error
if rec.Park {
to, detail = workflow.StatusParked, rec.ParkReason
n, err = q.ParkRecoveredTask(ctx, postgresdb.ParkRecoveredTaskParams{ID: id, ParkReason: rec.ParkReason})
} else {
n, err = q.RequeueRecoveredTask(ctx, postgresdb.RequeueRecoveredTaskParams{
ID: id, RetryMode: rec.RetryMode, ResumeFrom: rec.ResumeFrom, ResumeResults: rec.ResumeResults,
})
}
if err != nil {
return err
}
if n == 0 {
return storecontract.ErrStaleTransition
}
return recordTransition(ctx, q, postgresdb.InsertTransitionParams{
TaskID: id, FromStatus: workflow.StatusRunning, ToStatus: to, Detail: detail,
})
}
// OpenPRs returns tasks whose PR state should be reconciled with the forge,
+48
View File
@@ -0,0 +1,48 @@
package taskstate
import (
"testing"
)
func TestPlanRecovery(t *testing.T) {
tests := []struct {
name string
give RecoveryInput
want Recovery
}{
{
name: "resumes at the interrupted step with the finished steps' results",
give: RecoveryInput{Stage: "build", PriorResults: []byte(`{"plan":1}`)},
want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
},
{
name: "a crash inside the first resumed step keeps the row's resume point",
give: RecoveryInput{Stage: "build", ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
},
{
name: "the first step of a fresh attempt starts over",
give: RecoveryInput{Stage: "plan", RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)},
want: Recovery{RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)},
},
{
name: "no running step leaves the row as it is",
give: RecoveryInput{ResumeResults: []byte(`{}`)},
want: Recovery{ResumeResults: []byte(`{}`)},
},
{
name: "parks once the cap is spent",
give: RecoveryInput{Restarts: MaxRestartRequeues, Stage: "build", PriorResults: []byte(`{}`)},
want: Recovery{Park: true, ParkReason: "interrupted by 3 daemon restarts in a row; retry to run it again"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := PlanRecovery(tt.give)
if got.Park != tt.want.Park || got.ParkReason != tt.want.ParkReason || got.RetryMode != tt.want.RetryMode ||
got.ResumeFrom != tt.want.ResumeFrom || string(got.ResumeResults) != string(tt.want.ResumeResults) {
t.Errorf("PlanRecovery() = %+v, want %+v", got, tt.want)
}
})
}
}
+53
View File
@@ -356,3 +356,56 @@ func TransientRetryDue(class string, retries int, parkedAt, now time.Time) bool
}
return !now.Before(parkedAt.Add(transientBackoffBase << retries))
}
// MaxRestartRequeues is how many times daemon restarts may re-queue a task
// that was running before it parks for an operator.
const MaxRestartRequeues = 3
// RecoveryInput is what the store knows about a task a restart found running.
type RecoveryInput struct {
// Restarts is how many earlier restarts re-queued the task.
Restarts int
// Stage is the root step the restart interrupted, "" when none was running.
Stage string
// PriorResults are the step results recorded by the last step this attempt
// finished before Stage, nil when it finished none.
PriorResults []byte
// RetryMode, ResumeFrom and ResumeResults are the task row's own.
RetryMode, ResumeFrom string
ResumeResults []byte
}
// Recovery is how a task a restart found running continues.
type Recovery struct {
// Park reports that the task goes to an operator instead of the queue.
Park bool
ParkReason string
// RetryMode, ResumeFrom and ResumeResults are written to the re-queued row.
RetryMode, ResumeFrom string
ResumeResults []byte
}
// PlanRecovery decides how a restart treats a task left running: it resumes
// at the interrupted step in its kept worktree, with the results of the steps
// before it, rather than repaying them. A task already re-queued
// MaxRestartRequeues times parks instead.
func PlanRecovery(in RecoveryInput) Recovery {
if in.Restarts >= MaxRestartRequeues {
return Recovery{Park: true, ParkReason: fmt.Sprintf("interrupted by %d daemon restarts in a row; retry to run it again", in.Restarts)}
}
rec := Recovery{RetryMode: in.RetryMode, ResumeFrom: in.ResumeFrom, ResumeResults: in.ResumeResults}
if in.Stage == "" {
return rec
}
switch {
case in.PriorResults != nil:
rec.ResumeFrom, rec.ResumeResults = in.Stage, in.PriorResults
case in.ResumeFrom == in.Stage:
// The attempt was itself resumed at this step and finished none since.
default:
// The first step of a fresh attempt: nothing is finished, so start over.
return Recovery{RetryMode: in.RetryMode, ResumeResults: []byte("{}")}
}
rec.RetryMode = string(RetryKeepWorktree)
return rec
}