fix(store): a restart re-queues a running task at its interrupted step and parks it after 3 restarts (#1841)
deploy / plan (push) Successful in 7s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s
deploy / gate (push) Canceled after 4m4s

This commit is contained in:
sam committed 2026-11-09 14:36:08 +11:00
1 parent f44db54948
commit e75d5c8ade
11 files changed
+338 -62

No files matched your search

+39 -15
View File
@@ -2,6 +2,7 @@
package workflow package workflow
import ( import (
"bytes"
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
@@ -374,7 +375,8 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
park(ctx, tc, err.Error()) park(ctx, tc, err.Error())
return return
} }
for _, stage := range stages { from, history := tc.Task.ResumeFrom, tc.Task.ResumeHistory
for i, stage := range stages {
tc.Stage = stage.Name tc.Stage = stage.Name
if err := tc.Store.Update(ctx, t); err != nil { if err := tc.Store.Update(ctx, t); err != nil {
park(ctx, tc, fmt.Sprintf("persist task before stage %s: %v", stage.Name, err)) park(ctx, tc, fmt.Sprintf("persist task before stage %s: %v", stage.Name, err))
@@ -406,17 +408,7 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
// Shutdown is not a failure: record the step as interrupted and return // Shutdown is not a failure: record the step as interrupted and return
// without parking. // without parking.
if ctx.Err() != nil && err != nil { if ctx.Err() != nil && err != nil {
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) recordInterrupted(ctx, tc, stepID, stage.Name, err, stageLog)
finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{
StepID: stepID, ExecutionID: t.ID,
From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(),
})
cancel()
if recordErr != nil {
stageLog.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage.Name, "err", recordErr)
} else {
publishEvent(ctx, tc, finishEvent)
}
stageLog.InfoContext(ctx, "stage interrupted", "err", err) stageLog.InfoContext(ctx, "stage interrupted", "err", err)
return return
} }
@@ -435,6 +427,9 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
return return
} }
publishEvent(ctx, tc, finishEvent) publishEvent(ctx, tc, finishEvent)
if i == 0 {
tc.consumeResume(from, history)
}
if ended := endsRun(ctx, tc, stage, err, log, stageLog); ended { if ended := endsRun(ctx, tc, stage, err, log, stageLog); ended {
return return
@@ -445,6 +440,22 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
finish(ctx, tc, log) finish(ctx, tc, log)
} }
// recordInterrupted records a stage that shutdown stopped, on a context the
// shutdown has not cancelled.
func recordInterrupted(ctx context.Context, tc *TaskContext, stepID int64, stage string, err error, log *slog.Logger) {
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{
StepID: stepID, ExecutionID: tc.Task.ID,
From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(),
})
cancel()
if recordErr != nil {
log.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage, "err", recordErr)
} else {
publishEvent(ctx, tc, finishEvent)
}
}
// stageOutcome is how a stage that ran to its end records: a stage whose when // stageOutcome is how a stage that ran to its end records: a stage whose when
// was false is skipped, with the condition as its detail; an error fails it; // was false is skipped, with the condition as its detail; an error fails it;
// anything else succeeds. Interruption is the caller's call, because it owns // anything else succeeds. Interruption is the caller's call, because it owns
@@ -479,11 +490,9 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
if from == "" { if from == "" {
return stages, nil return stages, nil
} }
// The point is consumed: a run that stops again records its own, and a
// plain requeue after this one starts from the first step.
defer func() { tc.Task.ResumeFrom, tc.Task.ResumeResults, tc.Task.ResumeHistory = "", nil, nil }()
at := slices.IndexFunc(stages, func(s Stage) bool { return s.Name == from }) at := slices.IndexFunc(stages, func(s Stage) bool { return s.Name == from })
if at < 0 { if at < 0 {
tc.consumeResume(from, tc.Task.ResumeHistory)
return nil, fmt.Errorf("resume point %q is not a step of this workflow", from) return nil, fmt.Errorf("resume point %q is not a step of this workflow", from)
} }
if len(tc.Task.ResumeResults) > 0 { if len(tc.Task.ResumeResults) > 0 {
@@ -500,6 +509,21 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
return stages[at:], nil return stages[at:], nil
} }
// consumeResume clears the resume point. Run does it once the first resumed
// stage has finished, not when the point is read: until then the row must
// keep the point so a daemon crash inside that stage resumes there again,
// and a run that stops later records its own.
// A conversation the stage itself stopped with is a new one and stays.
func (tc *TaskContext) consumeResume(from string, history json.RawMessage) {
if from == "" {
return
}
tc.Task.ResumeFrom, tc.Task.ResumeResults = "", nil
if bytes.Equal(tc.Task.ResumeHistory, history) {
tc.Task.ResumeHistory = nil
}
}
// endsRun applies a recorded stage's result: a failure parks unless the stage // endsRun applies a recorded stage's result: a failure parks unless the stage
// continues on failure, and an outcome finishes the run. // continues on failure, and an outcome finishes the run.
func endsRun(ctx context.Context, tc *TaskContext, stage Stage, err error, log, stageLog *slog.Logger) bool { func endsRun(ctx context.Context, tc *TaskContext, stage Stage, err error, log, stageLog *slog.Logger) bool {
@@ -0,0 +1,10 @@
-- tasks.restart_requeues counts the daemon restarts that re-queued the task
-- while it ran; at the cap the task parks, and an operator retry resets it.
-- +goose Up
ALTER TABLE tasks ADD COLUMN restart_requeues bigint NOT NULL DEFAULT 0;
-- +goose Down
ALTER TABLE tasks DROP COLUMN restart_requeues;
@@ -24,7 +24,7 @@ func (q *Queries) CallStatusDetail(ctx context.Context, taskID int64) (string, e
const enqueueCallTask = `-- name: EnqueueCallTask :one const enqueueCallTask = `-- name: EnqueueCallTask :one
WITH caller AS ( WITH caller AS (
SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history FROM tasks t WHERE t.id = $1 FOR UPDATE SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history, t.restart_requeues FROM tasks t WHERE t.id = $1 FOR UPDATE
), number AS ( ), number AS (
INSERT INTO chat_issue_numbers (owner, repo, last_number) INSERT INTO chat_issue_numbers (owner, repo, last_number)
SELECT caller.owner, caller.repo, $9::bigint + 1 FROM caller SELECT caller.owner, caller.repo, $9::bigint + 1 FROM caller
@@ -43,7 +43,7 @@ SELECT
FROM caller, number FROM caller, number
WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
` `
type EnqueueCallTaskParams struct { type EnqueueCallTaskParams struct {
@@ -140,6 +140,7 @@ func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
@@ -587,6 +587,7 @@ type Task struct {
HandoffFromTaskID pgtype.Int8 HandoffFromTaskID pgtype.Int8
HandoffToTaskID pgtype.Int8 HandoffToTaskID pgtype.Int8
ResumeHistory []byte ResumeHistory []byte
RestartRequeues int64
} }
type ToolCall struct { type ToolCall struct {
@@ -121,7 +121,7 @@ const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many
UPDATE step_executions UPDATE step_executions
SET status = 'interrupted', finished_at = now() SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running' WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name RETURNING id, name, depth, kind
` `
type InterruptAttemptStepsParams struct { type InterruptAttemptStepsParams struct {
@@ -130,8 +130,10 @@ type InterruptAttemptStepsParams struct {
} }
type InterruptAttemptStepsRow struct { type InterruptAttemptStepsRow struct {
ID int64 ID int64
Name string Name string
Depth int32
Kind string
} }
// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md): // The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
@@ -148,7 +150,12 @@ func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemp
var items []InterruptAttemptStepsRow var items []InterruptAttemptStepsRow
for rows.Next() { for rows.Next() {
var i InterruptAttemptStepsRow var i InterruptAttemptStepsRow
if err := rows.Scan(&i.ID, &i.Name); err != nil { if err := rows.Scan(
&i.ID,
&i.Name,
&i.Depth,
&i.Kind,
); err != nil {
return nil, err return nil, err
} }
items = append(items, i) items = append(items, i)
@@ -299,19 +306,24 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu
} }
const lockRunningExecutions = `-- name: LockRunningExecutions :many const lockRunningExecutions = `-- name: LockRunningExecutions :many
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt,
restart_requeues, retry_mode, resume_from, resume_results
FROM tasks WHERE status = 'running' FOR UPDATE FROM tasks WHERE status = 'running' FOR UPDATE
` `
type LockRunningExecutionsRow struct { type LockRunningExecutionsRow struct {
ID int64 ID int64
OrgID string OrgID string
WorkspaceID string WorkspaceID string
Owner string Owner string
Repo string Repo string
IssueNumber int64 IssueNumber int64
Workflow string Workflow string
Attempt int64 Attempt int64
RestartRequeues int64
RetryMode string
ResumeFrom string
ResumeResults []byte
} }
// The executions a crashed or replaced daemon left running, locked for the // The executions a crashed or replaced daemon left running, locked for the
@@ -335,6 +347,10 @@ func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecu
&i.IssueNumber, &i.IssueNumber,
&i.Workflow, &i.Workflow,
&i.Attempt, &i.Attempt,
&i.RestartRequeues,
&i.RetryMode,
&i.ResumeFrom,
&i.ResumeResults,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@@ -395,6 +411,29 @@ func (q *Queries) LockStepExecution(ctx context.Context, id int64) (LockStepExec
return i, err return i, err
} }
const priorStageResults = `-- name: PriorStageResults :one
SELECT results FROM step_executions
WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage'
AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL
ORDER BY id DESC
LIMIT 1
`
type PriorStageResultsParams struct {
ExecutionID int64
Attempt int64
ID int64
}
// What the attempt's last finished root stage before the given step recorded,
// the results a restart resumes that step with.
func (q *Queries) PriorStageResults(ctx context.Context, arg PriorStageResultsParams) ([]byte, error) {
row := q.db.QueryRow(ctx, priorStageResults, arg.ExecutionID, arg.Attempt, arg.ID)
var results []byte
err := row.Scan(&results)
return results, err
}
const startStepExecution = `-- name: StartStepExecution :execrows const startStepExecution = `-- name: StartStepExecution :execrows
UPDATE step_executions SET status = 'running', started_at = now() UPDATE step_executions SET status = 'running', started_at = now()
WHERE id = $1 AND status = 'pending' WHERE id = $1 AND status = 'pending'
@@ -13,7 +13,7 @@ import (
) )
const activeTasksByOrigin = `-- name: ActiveTasksByOrigin :many const activeTasksByOrigin = `-- name: ActiveTasksByOrigin :many
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks
WHERE origin = $1 AND origin <> '' AND status IN ('queued', 'running') AND ($2::text IS NULL OR org_id = $2) WHERE origin = $1 AND origin <> '' AND status IN ('queued', 'running') AND ($2::text IS NULL OR org_id = $2)
ORDER BY id ORDER BY id
` `
@@ -88,6 +88,7 @@ func (q *Queries) ActiveTasksByOrigin(ctx context.Context, arg ActiveTasksByOrig
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@@ -167,7 +168,7 @@ func (q *Queries) BeginRemediationTask(ctx context.Context, arg BeginRemediation
const claimByIssue = `-- name: ClaimByIssue :one const claimByIssue = `-- name: ClaimByIssue :one
UPDATE tasks SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now() UPDATE tasks SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now()
WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND status = 'queued' WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND status = 'queued'
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
` `
type ClaimByIssueParams struct { type ClaimByIssueParams struct {
@@ -236,6 +237,7 @@ func (q *Queries) ClaimByIssue(ctx context.Context, arg ClaimByIssueParams) (Tas
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
@@ -250,7 +252,7 @@ WHERE id = (
FOR UPDATE SKIP LOCKED FOR UPDATE SKIP LOCKED
LIMIT 1 LIMIT 1
) )
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
` `
// FOR UPDATE SKIP LOCKED replaces the SQLite single-writer assumption: two // FOR UPDATE SKIP LOCKED replaces the SQLite single-writer assumption: two
@@ -317,6 +319,7 @@ func (q *Queries) ClaimNextTask(ctx context.Context) (Task, error) {
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
@@ -436,7 +439,7 @@ FROM number, (SELECT COALESCE(
'org-sys' 'org-sys'
) AS id) AS owner_org ) AS id) AS owner_org
WHERE $11::text IS NULL OR owner_org.id = $11 WHERE $11::text IS NULL OR owner_org.id = $11
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
` `
type InsertChatTaskParams struct { type InsertChatTaskParams struct {
@@ -537,6 +540,7 @@ func (q *Queries) InsertChatTask(ctx context.Context, arg InsertChatTaskParams)
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
@@ -549,7 +553,7 @@ SELECT $1, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, $2, t.sou
t.org_id, t.workspace_id, t.inputs, t.plan, t.plan_document, t.binding_id, t.binding_version, t.binding_signature, t.org_id, t.workspace_id, t.inputs, t.plan, t.plan_document, t.binding_id, t.binding_version, t.binding_signature,
t.binding_addr, t.origin, t.call_parent_task_id, t.call_depth, t.id t.binding_addr, t.origin, t.call_parent_task_id, t.call_depth, t.id
FROM tasks t WHERE t.id = $3 FROM tasks t WHERE t.id = $3
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
` `
type InsertHandoffTaskParams struct { type InsertHandoffTaskParams struct {
@@ -618,6 +622,7 @@ func (q *Queries) InsertHandoffTask(ctx context.Context, arg InsertHandoffTaskPa
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
@@ -908,6 +913,24 @@ func (q *Queries) NextTaskID(ctx context.Context) (int64, error) {
return column_1, err return column_1, err
} }
const parkRecoveredTask = `-- name: ParkRecoveredTask :execrows
UPDATE tasks SET status = 'parked', park_reason = $1, park_class = 'needs_decision', updated_at = now()
WHERE id = $2 AND status = 'running'
`
type ParkRecoveredTaskParams struct {
ParkReason string
ID int64
}
func (q *Queries) ParkRecoveredTask(ctx context.Context, arg ParkRecoveredTaskParams) (int64, error) {
result, err := q.db.Exec(ctx, parkRecoveredTask, arg.ParkReason, arg.ID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const parkTask = `-- name: ParkTask :execrows const parkTask = `-- name: ParkTask :execrows
UPDATE tasks UPDATE tasks
SET status = 'parked', park_reason = $2, park_class = $3, park_blocker = $5, updated_at = now() SET status = 'parked', park_reason = $2, park_class = $3, park_blocker = $5, updated_at = now()
@@ -957,12 +980,31 @@ func (q *Queries) PromotePendingReview(ctx context.Context, id int64) (int64, er
return result.RowsAffected(), nil return result.RowsAffected(), nil
} }
const recoverStaleTasks = `-- name: RecoverStaleTasks :execrows const requeueRecoveredTask = `-- name: RequeueRecoveredTask :execrows
UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running' UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1,
retry_mode = $1::text, resume_results = $2::jsonb,
resume_history = CASE WHEN resume_from = $3::text THEN resume_history ELSE 'null' END,
resume_from = $3::text, updated_at = now()
WHERE id = $4 AND status = 'running'
` `
func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) { type RequeueRecoveredTaskParams struct {
result, err := q.db.Exec(ctx, recoverStaleTasks) RetryMode string
ResumeResults []byte
ResumeFrom string
ID int64
}
// The restart's re-queue of a task it found running, resuming where the
// interrupted attempt stood. The conversation is kept only for the step the
// row already resumed at.
func (q *Queries) RequeueRecoveredTask(ctx context.Context, arg RequeueRecoveredTaskParams) (int64, error) {
result, err := q.db.Exec(ctx, requeueRecoveredTask,
arg.RetryMode,
arg.ResumeResults,
arg.ResumeFrom,
arg.ID,
)
if err != nil { if err != nil {
return 0, err return 0, err
} }
@@ -972,7 +1014,7 @@ func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) {
const requeueTask = `-- name: RequeueTask :execrows const requeueTask = `-- name: RequeueTask :execrows
UPDATE tasks SET status = 'queued', UPDATE tasks SET status = 'queued',
workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END, workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END,
park_reason = '', park_class = 'needs_decision', updated_at = now() park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now()
WHERE id = $2 AND status = $3 WHERE id = $2 AND status = $3
` `
@@ -1042,7 +1084,7 @@ func (q *Queries) RespondReviewGateTask(ctx context.Context, arg RespondReviewGa
} }
const retryTask = `-- name: RetryTask :execrows const retryTask = `-- name: RetryTask :execrows
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0,
workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END, workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END,
retry_mode = $2::text, retry_mode = $2::text,
resume_from = $3::text, resume_results = $4::jsonb, resume_history = 'null', resume_from = $3::text, resume_results = $4::jsonb, resume_history = 'null',
@@ -1142,7 +1184,7 @@ func (q *Queries) StampTaskBinding(ctx context.Context, arg StampTaskBindingPara
} }
const taskByID = `-- name: TaskByID :one const taskByID = `-- name: TaskByID :one
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2) SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2)
` `
type TaskByIDParams struct { type TaskByIDParams struct {
@@ -1208,12 +1250,13 @@ func (q *Queries) TaskByID(ctx context.Context, arg TaskByIDParams) (Task, error
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
const taskByIssue = `-- name: TaskByIssue :one const taskByIssue = `-- name: TaskByIssue :one
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL
AND ($4::text IS NULL OR org_id = $4) AND ($4::text IS NULL OR org_id = $4)
` `
@@ -1287,12 +1330,13 @@ func (q *Queries) TaskByIssue(ctx context.Context, arg TaskByIssueParams) (Task,
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
const taskByPR = `-- name: TaskByPR :one const taskByPR = `-- name: TaskByPR :one
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4 SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4
AND ($5::text IS NULL OR org_id = $5) AND ($5::text IS NULL OR org_id = $5)
` `
@@ -1371,6 +1415,7 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
) )
return i, err return i, err
} }
@@ -1423,7 +1468,7 @@ func (q *Queries) TransitionTask(ctx context.Context, arg TransitionTaskParams)
} }
const unpostedOutcomes = `-- name: UnpostedOutcomes :many const unpostedOutcomes = `-- name: UnpostedOutcomes :many
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks
WHERE origin <> '' AND outcome_posted_status <> status WHERE origin <> '' AND outcome_posted_status <> status
AND status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed', 'waiting_approval', 'pr_open') AND status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed', 'waiting_approval', 'pr_open')
AND handoff_to_task_id IS NULL AND handoff_to_task_id IS NULL
@@ -1498,6 +1543,7 @@ func (q *Queries) UnpostedOutcomes(ctx context.Context, scopeOrg pgtype.Text) ([
&i.HandoffFromTaskID, &i.HandoffFromTaskID,
&i.HandoffToTaskID, &i.HandoffToTaskID,
&i.ResumeHistory, &i.ResumeHistory,
&i.RestartRequeues,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@@ -56,7 +56,8 @@ RETURNING id, name;
-- The executions a crashed or replaced daemon left running, locked for the -- The executions a crashed or replaced daemon left running, locked for the
-- recovery transaction so the step sweep and the requeue cannot race a -- recovery transaction so the step sweep and the requeue cannot race a
-- concurrent write. The row carries the identity the steps' events need. -- concurrent write. The row carries the identity the steps' events need.
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt,
restart_requeues, retry_mode, resume_from, resume_results
FROM tasks WHERE status = 'running' FOR UPDATE; FROM tasks WHERE status = 'running' FOR UPDATE;
-- name: InterruptAttemptSteps :many -- name: InterruptAttemptSteps :many
@@ -68,7 +69,7 @@ FROM tasks WHERE status = 'running' FOR UPDATE;
UPDATE step_executions UPDATE step_executions
SET status = 'interrupted', finished_at = now() SET status = 'interrupted', finished_at = now()
WHERE execution_id = $1 AND attempt = $2 AND status = 'running' WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
RETURNING id, name; RETURNING id, name, depth, kind;
-- name: ListStepExecutions :many -- name: ListStepExecutions :many
-- attempt = 0 lists every attempt of the execution, oldest first; the index -- attempt = 0 lists every attempt of the execution, oldest first; the index
@@ -89,6 +90,15 @@ SELECT id, called_execution_id FROM step_executions
WHERE execution_id = $1 AND attempt = $2 WHERE execution_id = $1 AND attempt = $2
AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0; AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0;
-- name: PriorStageResults :one
-- What the attempt's last finished root stage before the given step recorded,
-- the results a restart resumes that step with.
SELECT results FROM step_executions
WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage'
AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL
ORDER BY id DESC
LIMIT 1;
-- name: LatestStageResults :one -- name: LatestStageResults :one
-- The most recent finished run of a root stage, across attempts: a resumed -- The most recent finished run of a root stage, across attempts: a resumed
-- attempt skips the stages before its resume point, so the last attempt that -- attempt skips the stages before its resume point, so the last attempt that
@@ -188,8 +188,19 @@ DELETE FROM tasks
WHERE status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed') WHERE status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed')
AND (sqlc.narg(scope_org)::text IS NULL OR org_id = sqlc.narg(scope_org)); AND (sqlc.narg(scope_org)::text IS NULL OR org_id = sqlc.narg(scope_org));
-- name: RecoverStaleTasks :execrows -- name: RequeueRecoveredTask :execrows
UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running'; -- The restart's re-queue of a task it found running, resuming where the
-- interrupted attempt stood. The conversation is kept only for the step the
-- row already resumed at.
UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1,
retry_mode = @retry_mode::text, resume_results = @resume_results::jsonb,
resume_history = CASE WHEN resume_from = @resume_from::text THEN resume_history ELSE 'null' END,
resume_from = @resume_from::text, updated_at = now()
WHERE id = @id AND status = 'running';
-- name: ParkRecoveredTask :execrows
UPDATE tasks SET status = 'parked', park_reason = @park_reason, park_class = 'needs_decision', updated_at = now()
WHERE id = @id AND status = 'running';
-- name: EnqueueIssue :execrows -- name: EnqueueIssue :execrows
-- The org is derived from the identity at the one place a task row is -- The org is derived from the identity at the one place a task row is
@@ -265,11 +276,11 @@ WHERE id = $1 AND status = 'pr_open';
-- name: RequeueTask :execrows -- name: RequeueTask :execrows
UPDATE tasks SET status = 'queued', UPDATE tasks SET status = 'queued',
workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END, workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END,
park_reason = '', park_class = 'needs_decision', updated_at = now() park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now()
WHERE id = @id AND status = @from_status; WHERE id = @id AND status = @from_status;
-- name: RetryTask :execrows -- name: RetryTask :execrows
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0,
workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END, workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END,
retry_mode = @retry_mode::text, retry_mode = @retry_mode::text,
resume_from = @resume_from::text, resume_results = @resume_results::jsonb, resume_history = 'null', resume_from = @resume_from::text, resume_results = @resume_results::jsonb, resume_history = 'null',
+44 -11
View File
@@ -702,8 +702,9 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
return eventID, nil return eventID, nil
} }
// RecoverStale requeues tasks left running and marks their running steps // RecoverStale requeues tasks left running, resuming each at the step the
// interrupted, in one transaction. // restart interrupted, and marks their running steps interrupted, in one
// transaction. A task already requeued too often parks instead.
func (s *Store) RecoverStale(ctx context.Context) (int64, error) { func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
tx, err := s.pool.Begin(ctx) tx, err := s.pool.Begin(ctx)
if err != nil { if err != nil {
@@ -723,28 +724,60 @@ func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
if err != nil { if err != nil {
return 0, err return 0, err
} }
in := taskstate.RecoveryInput{
Restarts: int(execution.RestartRequeues), RetryMode: execution.RetryMode,
ResumeFrom: execution.ResumeFrom, ResumeResults: execution.ResumeResults,
}
for _, step := range steps { for _, step := range steps {
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID, if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow, execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil { int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil {
return 0, err return 0, err
} }
if step.Depth != 0 || step.Kind != task.StepKindStage {
continue
}
in.Stage = step.Name
prior, err := q.PriorStageResults(ctx, postgresdb.PriorStageResultsParams{
ExecutionID: execution.ID, Attempt: execution.Attempt, ID: step.ID,
})
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return 0, err
}
in.PriorResults = prior
} }
if err := recordTransition(ctx, q, postgresdb.InsertTransitionParams{ if err := recoverTask(ctx, q, execution.ID, taskstate.PlanRecovery(in)); err != nil {
TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued,
Detail: "re-queued after a crashed or replaced daemon",
}); err != nil {
return 0, err return 0, err
} }
} }
n, err := q.RecoverStaleTasks(ctx)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil { if err := tx.Commit(ctx); err != nil {
return 0, err return 0, err
} }
return n, nil return int64(len(executions)), nil
}
// recoverTask writes one restart-found task's recovery and its transition.
func recoverTask(ctx context.Context, q *postgresdb.Queries, id int64, rec taskstate.Recovery) error {
to, detail := workflow.StatusQueued, "re-queued after a crashed or replaced daemon"
var n int64
var err error
if rec.Park {
to, detail = workflow.StatusParked, rec.ParkReason
n, err = q.ParkRecoveredTask(ctx, postgresdb.ParkRecoveredTaskParams{ID: id, ParkReason: rec.ParkReason})
} else {
n, err = q.RequeueRecoveredTask(ctx, postgresdb.RequeueRecoveredTaskParams{
ID: id, RetryMode: rec.RetryMode, ResumeFrom: rec.ResumeFrom, ResumeResults: rec.ResumeResults,
})
}
if err != nil {
return err
}
if n == 0 {
return storecontract.ErrStaleTransition
}
return recordTransition(ctx, q, postgresdb.InsertTransitionParams{
TaskID: id, FromStatus: workflow.StatusRunning, ToStatus: to, Detail: detail,
})
} }
// OpenPRs returns tasks whose PR state should be reconciled with the forge, // OpenPRs returns tasks whose PR state should be reconciled with the forge,
+48
View File
@@ -0,0 +1,48 @@
package taskstate
import (
"testing"
)
func TestPlanRecovery(t *testing.T) {
tests := []struct {
name string
give RecoveryInput
want Recovery
}{
{
name: "resumes at the interrupted step with the finished steps' results",
give: RecoveryInput{Stage: "build", PriorResults: []byte(`{"plan":1}`)},
want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
},
{
name: "a crash inside the first resumed step keeps the row's resume point",
give: RecoveryInput{Stage: "build", ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
},
{
name: "the first step of a fresh attempt starts over",
give: RecoveryInput{Stage: "plan", RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)},
want: Recovery{RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)},
},
{
name: "no running step leaves the row as it is",
give: RecoveryInput{ResumeResults: []byte(`{}`)},
want: Recovery{ResumeResults: []byte(`{}`)},
},
{
name: "parks once the cap is spent",
give: RecoveryInput{Restarts: MaxRestartRequeues, Stage: "build", PriorResults: []byte(`{}`)},
want: Recovery{Park: true, ParkReason: "interrupted by 3 daemon restarts in a row; retry to run it again"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := PlanRecovery(tt.give)
if got.Park != tt.want.Park || got.ParkReason != tt.want.ParkReason || got.RetryMode != tt.want.RetryMode ||
got.ResumeFrom != tt.want.ResumeFrom || string(got.ResumeResults) != string(tt.want.ResumeResults) {
t.Errorf("PlanRecovery() = %+v, want %+v", got, tt.want)
}
})
}
}
+53
View File
@@ -356,3 +356,56 @@ func TransientRetryDue(class string, retries int, parkedAt, now time.Time) bool
} }
return !now.Before(parkedAt.Add(transientBackoffBase << retries)) return !now.Before(parkedAt.Add(transientBackoffBase << retries))
} }
// MaxRestartRequeues is how many times daemon restarts may re-queue a task
// that was running before it parks for an operator.
const MaxRestartRequeues = 3
// RecoveryInput is what the store knows about a task a restart found running.
type RecoveryInput struct {
// Restarts is how many earlier restarts re-queued the task.
Restarts int
// Stage is the root step the restart interrupted, "" when none was running.
Stage string
// PriorResults are the step results recorded by the last step this attempt
// finished before Stage, nil when it finished none.
PriorResults []byte
// RetryMode, ResumeFrom and ResumeResults are the task row's own.
RetryMode, ResumeFrom string
ResumeResults []byte
}
// Recovery is how a task a restart found running continues.
type Recovery struct {
// Park reports that the task goes to an operator instead of the queue.
Park bool
ParkReason string
// RetryMode, ResumeFrom and ResumeResults are written to the re-queued row.
RetryMode, ResumeFrom string
ResumeResults []byte
}
// PlanRecovery decides how a restart treats a task left running: it resumes
// at the interrupted step in its kept worktree, with the results of the steps
// before it, rather than repaying them. A task already re-queued
// MaxRestartRequeues times parks instead.
func PlanRecovery(in RecoveryInput) Recovery {
if in.Restarts >= MaxRestartRequeues {
return Recovery{Park: true, ParkReason: fmt.Sprintf("interrupted by %d daemon restarts in a row; retry to run it again", in.Restarts)}
}
rec := Recovery{RetryMode: in.RetryMode, ResumeFrom: in.ResumeFrom, ResumeResults: in.ResumeResults}
if in.Stage == "" {
return rec
}
switch {
case in.PriorResults != nil:
rec.ResumeFrom, rec.ResumeResults = in.Stage, in.PriorResults
case in.ResumeFrom == in.Stage:
// The attempt was itself resumed at this step and finished none since.
default:
// The first step of a fresh attempt: nothing is finished, so start over.
return Recovery{RetryMode: in.RetryMode, ResumeResults: []byte("{}")}
}
rec.RetryMode = string(RetryKeepWorktree)
return rec
}