fix(store): a restart re-queues a running task at its interrupted step and parks it after 3 restarts (#1841)
This commit is contained in:
11 files changed
+338
-62
No files matched your search
@@ -2,6 +2,7 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
@@ -374,7 +375,8 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
|
||||
park(ctx, tc, err.Error())
|
||||
return
|
||||
}
|
||||
for _, stage := range stages {
|
||||
from, history := tc.Task.ResumeFrom, tc.Task.ResumeHistory
|
||||
for i, stage := range stages {
|
||||
tc.Stage = stage.Name
|
||||
if err := tc.Store.Update(ctx, t); err != nil {
|
||||
park(ctx, tc, fmt.Sprintf("persist task before stage %s: %v", stage.Name, err))
|
||||
@@ -406,17 +408,7 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
|
||||
// Shutdown is not a failure: record the step as interrupted and return
|
||||
// without parking.
|
||||
if ctx.Err() != nil && err != nil {
|
||||
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{
|
||||
StepID: stepID, ExecutionID: t.ID,
|
||||
From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(),
|
||||
})
|
||||
cancel()
|
||||
if recordErr != nil {
|
||||
stageLog.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage.Name, "err", recordErr)
|
||||
} else {
|
||||
publishEvent(ctx, tc, finishEvent)
|
||||
}
|
||||
recordInterrupted(ctx, tc, stepID, stage.Name, err, stageLog)
|
||||
stageLog.InfoContext(ctx, "stage interrupted", "err", err)
|
||||
return
|
||||
}
|
||||
@@ -435,6 +427,9 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
|
||||
return
|
||||
}
|
||||
publishEvent(ctx, tc, finishEvent)
|
||||
if i == 0 {
|
||||
tc.consumeResume(from, history)
|
||||
}
|
||||
|
||||
if ended := endsRun(ctx, tc, stage, err, log, stageLog); ended {
|
||||
return
|
||||
@@ -445,6 +440,22 @@ func Run(ctx context.Context, wf Workflow, tc *TaskContext) {
|
||||
finish(ctx, tc, log)
|
||||
}
|
||||
|
||||
// recordInterrupted records a stage that shutdown stopped, on a context the
|
||||
// shutdown has not cancelled.
|
||||
func recordInterrupted(ctx context.Context, tc *TaskContext, stepID int64, stage string, err error, log *slog.Logger) {
|
||||
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
finishEvent, recordErr := tc.Store.FinishStep(finishCtx, StepFinish{
|
||||
StepID: stepID, ExecutionID: tc.Task.ID,
|
||||
From: taskstate.StepRunning, To: taskstate.StepInterrupted, Detail: err.Error(),
|
||||
})
|
||||
cancel()
|
||||
if recordErr != nil {
|
||||
log.WarnContext(ctx, "interrupted step finish could not be recorded", "stage", stage, "err", recordErr)
|
||||
} else {
|
||||
publishEvent(ctx, tc, finishEvent)
|
||||
}
|
||||
}
|
||||
|
||||
// stageOutcome is how a stage that ran to its end records: a stage whose when
|
||||
// was false is skipped, with the condition as its detail; an error fails it;
|
||||
// anything else succeeds. Interruption is the caller's call, because it owns
|
||||
@@ -479,11 +490,9 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
|
||||
if from == "" {
|
||||
return stages, nil
|
||||
}
|
||||
// The point is consumed: a run that stops again records its own, and a
|
||||
// plain requeue after this one starts from the first step.
|
||||
defer func() { tc.Task.ResumeFrom, tc.Task.ResumeResults, tc.Task.ResumeHistory = "", nil, nil }()
|
||||
at := slices.IndexFunc(stages, func(s Stage) bool { return s.Name == from })
|
||||
if at < 0 {
|
||||
tc.consumeResume(from, tc.Task.ResumeHistory)
|
||||
return nil, fmt.Errorf("resume point %q is not a step of this workflow", from)
|
||||
}
|
||||
if len(tc.Task.ResumeResults) > 0 {
|
||||
@@ -500,6 +509,21 @@ func (tc *TaskContext) resumeStages(stages []Stage) ([]Stage, error) {
|
||||
return stages[at:], nil
|
||||
}
|
||||
|
||||
// consumeResume clears the resume point. Run does it once the first resumed
|
||||
// stage has finished, not when the point is read: until then the row must
|
||||
// keep the point so a daemon crash inside that stage resumes there again,
|
||||
// and a run that stops later records its own.
|
||||
// A conversation the stage itself stopped with is a new one and stays.
|
||||
func (tc *TaskContext) consumeResume(from string, history json.RawMessage) {
|
||||
if from == "" {
|
||||
return
|
||||
}
|
||||
tc.Task.ResumeFrom, tc.Task.ResumeResults = "", nil
|
||||
if bytes.Equal(tc.Task.ResumeHistory, history) {
|
||||
tc.Task.ResumeHistory = nil
|
||||
}
|
||||
}
|
||||
|
||||
// endsRun applies a recorded stage's result: a failure parks unless the stage
|
||||
// continues on failure, and an outcome finishes the run.
|
||||
func endsRun(ctx context.Context, tc *TaskContext, stage Stage, err error, log, stageLog *slog.Logger) bool {
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
-- tasks.restart_requeues counts the daemon restarts that re-queued the task
|
||||
-- while it ran; at the cap the task parks, and an operator retry resets it.
|
||||
|
||||
-- +goose Up
|
||||
|
||||
ALTER TABLE tasks ADD COLUMN restart_requeues bigint NOT NULL DEFAULT 0;
|
||||
|
||||
-- +goose Down
|
||||
|
||||
ALTER TABLE tasks DROP COLUMN restart_requeues;
|
||||
@@ -24,7 +24,7 @@ func (q *Queries) CallStatusDetail(ctx context.Context, taskID int64) (string, e
|
||||
|
||||
const enqueueCallTask = `-- name: EnqueueCallTask :one
|
||||
WITH caller AS (
|
||||
SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history FROM tasks t WHERE t.id = $1 FOR UPDATE
|
||||
SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.binding_signature, t.binding_addr, t.park_blocker, t.outcome_posted_status, t.plan_document, t.handoff_from_task_id, t.handoff_to_task_id, t.resume_history, t.restart_requeues FROM tasks t WHERE t.id = $1 FOR UPDATE
|
||||
), number AS (
|
||||
INSERT INTO chat_issue_numbers (owner, repo, last_number)
|
||||
SELECT caller.owner, caller.repo, $9::bigint + 1 FROM caller
|
||||
@@ -43,7 +43,7 @@ SELECT
|
||||
FROM caller, number
|
||||
WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int
|
||||
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
|
||||
`
|
||||
|
||||
type EnqueueCallTaskParams struct {
|
||||
@@ -140,6 +140,7 @@ func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -587,6 +587,7 @@ type Task struct {
|
||||
HandoffFromTaskID pgtype.Int8
|
||||
HandoffToTaskID pgtype.Int8
|
||||
ResumeHistory []byte
|
||||
RestartRequeues int64
|
||||
}
|
||||
|
||||
type ToolCall struct {
|
||||
|
||||
@@ -121,7 +121,7 @@ const interruptAttemptSteps = `-- name: InterruptAttemptSteps :many
|
||||
UPDATE step_executions
|
||||
SET status = 'interrupted', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
|
||||
RETURNING id, name
|
||||
RETURNING id, name, depth, kind
|
||||
`
|
||||
|
||||
type InterruptAttemptStepsParams struct {
|
||||
@@ -130,8 +130,10 @@ type InterruptAttemptStepsParams struct {
|
||||
}
|
||||
|
||||
type InterruptAttemptStepsRow struct {
|
||||
ID int64
|
||||
Name string
|
||||
ID int64
|
||||
Name string
|
||||
Depth int32
|
||||
Kind string
|
||||
}
|
||||
|
||||
// The crash-recovery step edge (docs/prds/execution-tree-state-machine.md):
|
||||
@@ -148,7 +150,12 @@ func (q *Queries) InterruptAttemptSteps(ctx context.Context, arg InterruptAttemp
|
||||
var items []InterruptAttemptStepsRow
|
||||
for rows.Next() {
|
||||
var i InterruptAttemptStepsRow
|
||||
if err := rows.Scan(&i.ID, &i.Name); err != nil {
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Name,
|
||||
&i.Depth,
|
||||
&i.Kind,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
@@ -299,19 +306,24 @@ func (q *Queries) LockExecutionForStep(ctx context.Context, id int64) (LockExecu
|
||||
}
|
||||
|
||||
const lockRunningExecutions = `-- name: LockRunningExecutions :many
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt,
|
||||
restart_requeues, retry_mode, resume_from, resume_results
|
||||
FROM tasks WHERE status = 'running' FOR UPDATE
|
||||
`
|
||||
|
||||
type LockRunningExecutionsRow struct {
|
||||
ID int64
|
||||
OrgID string
|
||||
WorkspaceID string
|
||||
Owner string
|
||||
Repo string
|
||||
IssueNumber int64
|
||||
Workflow string
|
||||
Attempt int64
|
||||
ID int64
|
||||
OrgID string
|
||||
WorkspaceID string
|
||||
Owner string
|
||||
Repo string
|
||||
IssueNumber int64
|
||||
Workflow string
|
||||
Attempt int64
|
||||
RestartRequeues int64
|
||||
RetryMode string
|
||||
ResumeFrom string
|
||||
ResumeResults []byte
|
||||
}
|
||||
|
||||
// The executions a crashed or replaced daemon left running, locked for the
|
||||
@@ -335,6 +347,10 @@ func (q *Queries) LockRunningExecutions(ctx context.Context) ([]LockRunningExecu
|
||||
&i.IssueNumber,
|
||||
&i.Workflow,
|
||||
&i.Attempt,
|
||||
&i.RestartRequeues,
|
||||
&i.RetryMode,
|
||||
&i.ResumeFrom,
|
||||
&i.ResumeResults,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -395,6 +411,29 @@ func (q *Queries) LockStepExecution(ctx context.Context, id int64) (LockStepExec
|
||||
return i, err
|
||||
}
|
||||
|
||||
const priorStageResults = `-- name: PriorStageResults :one
|
||||
SELECT results FROM step_executions
|
||||
WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage'
|
||||
AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
type PriorStageResultsParams struct {
|
||||
ExecutionID int64
|
||||
Attempt int64
|
||||
ID int64
|
||||
}
|
||||
|
||||
// What the attempt's last finished root stage before the given step recorded,
|
||||
// the results a restart resumes that step with.
|
||||
func (q *Queries) PriorStageResults(ctx context.Context, arg PriorStageResultsParams) ([]byte, error) {
|
||||
row := q.db.QueryRow(ctx, priorStageResults, arg.ExecutionID, arg.Attempt, arg.ID)
|
||||
var results []byte
|
||||
err := row.Scan(&results)
|
||||
return results, err
|
||||
}
|
||||
|
||||
const startStepExecution = `-- name: StartStepExecution :execrows
|
||||
UPDATE step_executions SET status = 'running', started_at = now()
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
const activeTasksByOrigin = `-- name: ActiveTasksByOrigin :many
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks
|
||||
WHERE origin = $1 AND origin <> '' AND status IN ('queued', 'running') AND ($2::text IS NULL OR org_id = $2)
|
||||
ORDER BY id
|
||||
`
|
||||
@@ -88,6 +88,7 @@ func (q *Queries) ActiveTasksByOrigin(ctx context.Context, arg ActiveTasksByOrig
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -167,7 +168,7 @@ func (q *Queries) BeginRemediationTask(ctx context.Context, arg BeginRemediation
|
||||
const claimByIssue = `-- name: ClaimByIssue :one
|
||||
UPDATE tasks SET status = 'running', attempt = attempt + 1, outputs = '', updated_at = now()
|
||||
WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND status = 'queued'
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
|
||||
`
|
||||
|
||||
type ClaimByIssueParams struct {
|
||||
@@ -236,6 +237,7 @@ func (q *Queries) ClaimByIssue(ctx context.Context, arg ClaimByIssueParams) (Tas
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -250,7 +252,7 @@ WHERE id = (
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
|
||||
`
|
||||
|
||||
// FOR UPDATE SKIP LOCKED replaces the SQLite single-writer assumption: two
|
||||
@@ -317,6 +319,7 @@ func (q *Queries) ClaimNextTask(ctx context.Context) (Task, error) {
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -436,7 +439,7 @@ FROM number, (SELECT COALESCE(
|
||||
'org-sys'
|
||||
) AS id) AS owner_org
|
||||
WHERE $11::text IS NULL OR owner_org.id = $11
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
|
||||
`
|
||||
|
||||
type InsertChatTaskParams struct {
|
||||
@@ -537,6 +540,7 @@ func (q *Queries) InsertChatTask(ctx context.Context, arg InsertChatTaskParams)
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -549,7 +553,7 @@ SELECT $1, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, $2, t.sou
|
||||
t.org_id, t.workspace_id, t.inputs, t.plan, t.plan_document, t.binding_id, t.binding_version, t.binding_signature,
|
||||
t.binding_addr, t.origin, t.call_parent_task_id, t.call_depth, t.id
|
||||
FROM tasks t WHERE t.id = $3
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues
|
||||
`
|
||||
|
||||
type InsertHandoffTaskParams struct {
|
||||
@@ -618,6 +622,7 @@ func (q *Queries) InsertHandoffTask(ctx context.Context, arg InsertHandoffTaskPa
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -908,6 +913,24 @@ func (q *Queries) NextTaskID(ctx context.Context) (int64, error) {
|
||||
return column_1, err
|
||||
}
|
||||
|
||||
const parkRecoveredTask = `-- name: ParkRecoveredTask :execrows
|
||||
UPDATE tasks SET status = 'parked', park_reason = $1, park_class = 'needs_decision', updated_at = now()
|
||||
WHERE id = $2 AND status = 'running'
|
||||
`
|
||||
|
||||
type ParkRecoveredTaskParams struct {
|
||||
ParkReason string
|
||||
ID int64
|
||||
}
|
||||
|
||||
func (q *Queries) ParkRecoveredTask(ctx context.Context, arg ParkRecoveredTaskParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, parkRecoveredTask, arg.ParkReason, arg.ID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const parkTask = `-- name: ParkTask :execrows
|
||||
UPDATE tasks
|
||||
SET status = 'parked', park_reason = $2, park_class = $3, park_blocker = $5, updated_at = now()
|
||||
@@ -957,12 +980,31 @@ func (q *Queries) PromotePendingReview(ctx context.Context, id int64) (int64, er
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const recoverStaleTasks = `-- name: RecoverStaleTasks :execrows
|
||||
UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running'
|
||||
const requeueRecoveredTask = `-- name: RequeueRecoveredTask :execrows
|
||||
UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1,
|
||||
retry_mode = $1::text, resume_results = $2::jsonb,
|
||||
resume_history = CASE WHEN resume_from = $3::text THEN resume_history ELSE 'null' END,
|
||||
resume_from = $3::text, updated_at = now()
|
||||
WHERE id = $4 AND status = 'running'
|
||||
`
|
||||
|
||||
func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, recoverStaleTasks)
|
||||
type RequeueRecoveredTaskParams struct {
|
||||
RetryMode string
|
||||
ResumeResults []byte
|
||||
ResumeFrom string
|
||||
ID int64
|
||||
}
|
||||
|
||||
// The restart's re-queue of a task it found running, resuming where the
|
||||
// interrupted attempt stood. The conversation is kept only for the step the
|
||||
// row already resumed at.
|
||||
func (q *Queries) RequeueRecoveredTask(ctx context.Context, arg RequeueRecoveredTaskParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, requeueRecoveredTask,
|
||||
arg.RetryMode,
|
||||
arg.ResumeResults,
|
||||
arg.ResumeFrom,
|
||||
arg.ID,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -972,7 +1014,7 @@ func (q *Queries) RecoverStaleTasks(ctx context.Context) (int64, error) {
|
||||
const requeueTask = `-- name: RequeueTask :execrows
|
||||
UPDATE tasks SET status = 'queued',
|
||||
workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END,
|
||||
park_reason = '', park_class = 'needs_decision', updated_at = now()
|
||||
park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now()
|
||||
WHERE id = $2 AND status = $3
|
||||
`
|
||||
|
||||
@@ -1042,7 +1084,7 @@ func (q *Queries) RespondReviewGateTask(ctx context.Context, arg RespondReviewGa
|
||||
}
|
||||
|
||||
const retryTask = `-- name: RetryTask :execrows
|
||||
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1,
|
||||
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0,
|
||||
workflow = CASE WHEN $1::text = '' THEN workflow ELSE $1::text END,
|
||||
retry_mode = $2::text,
|
||||
resume_from = $3::text, resume_results = $4::jsonb, resume_history = 'null',
|
||||
@@ -1142,7 +1184,7 @@ func (q *Queries) StampTaskBinding(ctx context.Context, arg StampTaskBindingPara
|
||||
}
|
||||
|
||||
const taskByID = `-- name: TaskByID :one
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2)
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE id = $1 AND ($2::text IS NULL OR org_id = $2)
|
||||
`
|
||||
|
||||
type TaskByIDParams struct {
|
||||
@@ -1208,12 +1250,13 @@ func (q *Queries) TaskByID(ctx context.Context, arg TaskByIDParams) (Task, error
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const taskByIssue = `-- name: TaskByIssue :one
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND issue_number = $3 AND handoff_to_task_id IS NULL
|
||||
AND ($4::text IS NULL OR org_id = $4)
|
||||
`
|
||||
|
||||
@@ -1287,12 +1330,13 @@ func (q *Queries) TaskByIssue(ctx context.Context, arg TaskByIssueParams) (Task,
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const taskByPR = `-- name: TaskByPR :one
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks WHERE owner = $1 AND repo = $2 AND pr_number = $3 AND status = $4
|
||||
AND ($5::text IS NULL OR org_id = $5)
|
||||
`
|
||||
|
||||
@@ -1371,6 +1415,7 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -1423,7 +1468,7 @@ func (q *Queries) TransitionTask(ctx context.Context, arg TransitionTaskParams)
|
||||
}
|
||||
|
||||
const unpostedOutcomes = `-- name: UnpostedOutcomes :many
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history FROM tasks
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, binding_signature, binding_addr, park_blocker, outcome_posted_status, plan_document, handoff_from_task_id, handoff_to_task_id, resume_history, restart_requeues FROM tasks
|
||||
WHERE origin <> '' AND outcome_posted_status <> status
|
||||
AND status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed', 'waiting_approval', 'pr_open')
|
||||
AND handoff_to_task_id IS NULL
|
||||
@@ -1498,6 +1543,7 @@ func (q *Queries) UnpostedOutcomes(ctx context.Context, scopeOrg pgtype.Text) ([
|
||||
&i.HandoffFromTaskID,
|
||||
&i.HandoffToTaskID,
|
||||
&i.ResumeHistory,
|
||||
&i.RestartRequeues,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -56,7 +56,8 @@ RETURNING id, name;
|
||||
-- The executions a crashed or replaced daemon left running, locked for the
|
||||
-- recovery transaction so the step sweep and the requeue cannot race a
|
||||
-- concurrent write. The row carries the identity the steps' events need.
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt
|
||||
SELECT id, org_id, workspace_id, owner, repo, issue_number, workflow, attempt,
|
||||
restart_requeues, retry_mode, resume_from, resume_results
|
||||
FROM tasks WHERE status = 'running' FOR UPDATE;
|
||||
|
||||
-- name: InterruptAttemptSteps :many
|
||||
@@ -68,7 +69,7 @@ FROM tasks WHERE status = 'running' FOR UPDATE;
|
||||
UPDATE step_executions
|
||||
SET status = 'interrupted', finished_at = now()
|
||||
WHERE execution_id = $1 AND attempt = $2 AND status = 'running'
|
||||
RETURNING id, name;
|
||||
RETURNING id, name, depth, kind;
|
||||
|
||||
-- name: ListStepExecutions :many
|
||||
-- attempt = 0 lists every attempt of the execution, oldest first; the index
|
||||
@@ -89,6 +90,15 @@ SELECT id, called_execution_id FROM step_executions
|
||||
WHERE execution_id = $1 AND attempt = $2
|
||||
AND kind = 'call' AND status IN ('pending', 'running') AND called_execution_id <> 0;
|
||||
|
||||
-- name: PriorStageResults :one
|
||||
-- What the attempt's last finished root stage before the given step recorded,
|
||||
-- the results a restart resumes that step with.
|
||||
SELECT results FROM step_executions
|
||||
WHERE execution_id = $1 AND attempt = $2 AND id < $3 AND depth = 0 AND kind = 'stage'
|
||||
AND status IN ('succeeded', 'failed', 'skipped') AND results IS NOT NULL
|
||||
ORDER BY id DESC
|
||||
LIMIT 1;
|
||||
|
||||
-- name: LatestStageResults :one
|
||||
-- The most recent finished run of a root stage, across attempts: a resumed
|
||||
-- attempt skips the stages before its resume point, so the last attempt that
|
||||
|
||||
@@ -188,8 +188,19 @@ DELETE FROM tasks
|
||||
WHERE status IN ('merged', 'rejected', 'dead', 'closed_wont_do', 'completed')
|
||||
AND (sqlc.narg(scope_org)::text IS NULL OR org_id = sqlc.narg(scope_org));
|
||||
|
||||
-- name: RecoverStaleTasks :execrows
|
||||
UPDATE tasks SET status = 'queued', updated_at = now() WHERE status = 'running';
|
||||
-- name: RequeueRecoveredTask :execrows
|
||||
-- The restart's re-queue of a task it found running, resuming where the
|
||||
-- interrupted attempt stood. The conversation is kept only for the step the
|
||||
-- row already resumed at.
|
||||
UPDATE tasks SET status = 'queued', restart_requeues = restart_requeues + 1,
|
||||
retry_mode = @retry_mode::text, resume_results = @resume_results::jsonb,
|
||||
resume_history = CASE WHEN resume_from = @resume_from::text THEN resume_history ELSE 'null' END,
|
||||
resume_from = @resume_from::text, updated_at = now()
|
||||
WHERE id = @id AND status = 'running';
|
||||
|
||||
-- name: ParkRecoveredTask :execrows
|
||||
UPDATE tasks SET status = 'parked', park_reason = @park_reason, park_class = 'needs_decision', updated_at = now()
|
||||
WHERE id = @id AND status = 'running';
|
||||
|
||||
-- name: EnqueueIssue :execrows
|
||||
-- The org is derived from the identity at the one place a task row is
|
||||
@@ -265,11 +276,11 @@ WHERE id = $1 AND status = 'pr_open';
|
||||
-- name: RequeueTask :execrows
|
||||
UPDATE tasks SET status = 'queued',
|
||||
workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END,
|
||||
park_reason = '', park_class = 'needs_decision', updated_at = now()
|
||||
park_reason = '', park_class = 'needs_decision', restart_requeues = 0, updated_at = now()
|
||||
WHERE id = @id AND status = @from_status;
|
||||
|
||||
-- name: RetryTask :execrows
|
||||
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1,
|
||||
UPDATE tasks SET status = 'queued', retry_count = retry_count + 1, restart_requeues = 0,
|
||||
workflow = CASE WHEN @workflow::text = '' THEN workflow ELSE @workflow::text END,
|
||||
retry_mode = @retry_mode::text,
|
||||
resume_from = @resume_from::text, resume_results = @resume_results::jsonb, resume_history = 'null',
|
||||
|
||||
@@ -702,8 +702,9 @@ func (s *Store) ArchiveTask(ctx context.Context, taskID int64, fromStatus string
|
||||
return eventID, nil
|
||||
}
|
||||
|
||||
// RecoverStale requeues tasks left running and marks their running steps
|
||||
// interrupted, in one transaction.
|
||||
// RecoverStale requeues tasks left running, resuming each at the step the
|
||||
// restart interrupted, and marks their running steps interrupted, in one
|
||||
// transaction. A task already requeued too often parks instead.
|
||||
func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
|
||||
tx, err := s.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
@@ -723,28 +724,60 @@ func (s *Store) RecoverStale(ctx context.Context) (int64, error) {
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
in := taskstate.RecoveryInput{
|
||||
Restarts: int(execution.RestartRequeues), RetryMode: execution.RetryMode,
|
||||
ResumeFrom: execution.ResumeFrom, ResumeResults: execution.ResumeResults,
|
||||
}
|
||||
for _, step := range steps {
|
||||
if _, err := insertEventQ(ctx, q, stepEvent(events.KindStageFinish, execution.ID,
|
||||
execution.Owner, execution.Repo, execution.IssueNumber, execution.Workflow,
|
||||
int(execution.Attempt), step.Name, map[string]any{"interrupted": true})); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if step.Depth != 0 || step.Kind != task.StepKindStage {
|
||||
continue
|
||||
}
|
||||
in.Stage = step.Name
|
||||
prior, err := q.PriorStageResults(ctx, postgresdb.PriorStageResultsParams{
|
||||
ExecutionID: execution.ID, Attempt: execution.Attempt, ID: step.ID,
|
||||
})
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return 0, err
|
||||
}
|
||||
in.PriorResults = prior
|
||||
}
|
||||
if err := recordTransition(ctx, q, postgresdb.InsertTransitionParams{
|
||||
TaskID: execution.ID, FromStatus: workflow.StatusRunning, ToStatus: workflow.StatusQueued,
|
||||
Detail: "re-queued after a crashed or replaced daemon",
|
||||
}); err != nil {
|
||||
if err := recoverTask(ctx, q, execution.ID, taskstate.PlanRecovery(in)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
n, err := q.RecoverStaleTasks(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return n, nil
|
||||
return int64(len(executions)), nil
|
||||
}
|
||||
|
||||
// recoverTask writes one restart-found task's recovery and its transition.
|
||||
func recoverTask(ctx context.Context, q *postgresdb.Queries, id int64, rec taskstate.Recovery) error {
|
||||
to, detail := workflow.StatusQueued, "re-queued after a crashed or replaced daemon"
|
||||
var n int64
|
||||
var err error
|
||||
if rec.Park {
|
||||
to, detail = workflow.StatusParked, rec.ParkReason
|
||||
n, err = q.ParkRecoveredTask(ctx, postgresdb.ParkRecoveredTaskParams{ID: id, ParkReason: rec.ParkReason})
|
||||
} else {
|
||||
n, err = q.RequeueRecoveredTask(ctx, postgresdb.RequeueRecoveredTaskParams{
|
||||
ID: id, RetryMode: rec.RetryMode, ResumeFrom: rec.ResumeFrom, ResumeResults: rec.ResumeResults,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return storecontract.ErrStaleTransition
|
||||
}
|
||||
return recordTransition(ctx, q, postgresdb.InsertTransitionParams{
|
||||
TaskID: id, FromStatus: workflow.StatusRunning, ToStatus: to, Detail: detail,
|
||||
})
|
||||
}
|
||||
|
||||
// OpenPRs returns tasks whose PR state should be reconciled with the forge,
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package taskstate
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPlanRecovery(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
give RecoveryInput
|
||||
want Recovery
|
||||
}{
|
||||
{
|
||||
name: "resumes at the interrupted step with the finished steps' results",
|
||||
give: RecoveryInput{Stage: "build", PriorResults: []byte(`{"plan":1}`)},
|
||||
want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
|
||||
},
|
||||
{
|
||||
name: "a crash inside the first resumed step keeps the row's resume point",
|
||||
give: RecoveryInput{Stage: "build", ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
|
||||
want: Recovery{RetryMode: string(RetryKeepWorktree), ResumeFrom: "build", ResumeResults: []byte(`{"plan":1}`)},
|
||||
},
|
||||
{
|
||||
name: "the first step of a fresh attempt starts over",
|
||||
give: RecoveryInput{Stage: "plan", RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)},
|
||||
want: Recovery{RetryMode: string(RetryContinuePushedWork), ResumeResults: []byte(`{}`)},
|
||||
},
|
||||
{
|
||||
name: "no running step leaves the row as it is",
|
||||
give: RecoveryInput{ResumeResults: []byte(`{}`)},
|
||||
want: Recovery{ResumeResults: []byte(`{}`)},
|
||||
},
|
||||
{
|
||||
name: "parks once the cap is spent",
|
||||
give: RecoveryInput{Restarts: MaxRestartRequeues, Stage: "build", PriorResults: []byte(`{}`)},
|
||||
want: Recovery{Park: true, ParkReason: "interrupted by 3 daemon restarts in a row; retry to run it again"},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := PlanRecovery(tt.give)
|
||||
if got.Park != tt.want.Park || got.ParkReason != tt.want.ParkReason || got.RetryMode != tt.want.RetryMode ||
|
||||
got.ResumeFrom != tt.want.ResumeFrom || string(got.ResumeResults) != string(tt.want.ResumeResults) {
|
||||
t.Errorf("PlanRecovery() = %+v, want %+v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -356,3 +356,56 @@ func TransientRetryDue(class string, retries int, parkedAt, now time.Time) bool
|
||||
}
|
||||
return !now.Before(parkedAt.Add(transientBackoffBase << retries))
|
||||
}
|
||||
|
||||
// MaxRestartRequeues is how many times daemon restarts may re-queue a task
|
||||
// that was running before it parks for an operator.
|
||||
const MaxRestartRequeues = 3
|
||||
|
||||
// RecoveryInput is what the store knows about a task a restart found running.
|
||||
type RecoveryInput struct {
|
||||
// Restarts is how many earlier restarts re-queued the task.
|
||||
Restarts int
|
||||
// Stage is the root step the restart interrupted, "" when none was running.
|
||||
Stage string
|
||||
// PriorResults are the step results recorded by the last step this attempt
|
||||
// finished before Stage, nil when it finished none.
|
||||
PriorResults []byte
|
||||
// RetryMode, ResumeFrom and ResumeResults are the task row's own.
|
||||
RetryMode, ResumeFrom string
|
||||
ResumeResults []byte
|
||||
}
|
||||
|
||||
// Recovery is how a task a restart found running continues.
|
||||
type Recovery struct {
|
||||
// Park reports that the task goes to an operator instead of the queue.
|
||||
Park bool
|
||||
ParkReason string
|
||||
// RetryMode, ResumeFrom and ResumeResults are written to the re-queued row.
|
||||
RetryMode, ResumeFrom string
|
||||
ResumeResults []byte
|
||||
}
|
||||
|
||||
// PlanRecovery decides how a restart treats a task left running: it resumes
|
||||
// at the interrupted step in its kept worktree, with the results of the steps
|
||||
// before it, rather than repaying them. A task already re-queued
|
||||
// MaxRestartRequeues times parks instead.
|
||||
func PlanRecovery(in RecoveryInput) Recovery {
|
||||
if in.Restarts >= MaxRestartRequeues {
|
||||
return Recovery{Park: true, ParkReason: fmt.Sprintf("interrupted by %d daemon restarts in a row; retry to run it again", in.Restarts)}
|
||||
}
|
||||
rec := Recovery{RetryMode: in.RetryMode, ResumeFrom: in.ResumeFrom, ResumeResults: in.ResumeResults}
|
||||
if in.Stage == "" {
|
||||
return rec
|
||||
}
|
||||
switch {
|
||||
case in.PriorResults != nil:
|
||||
rec.ResumeFrom, rec.ResumeResults = in.Stage, in.PriorResults
|
||||
case in.ResumeFrom == in.Stage:
|
||||
// The attempt was itself resumed at this step and finished none since.
|
||||
default:
|
||||
// The first step of a fresh attempt: nothing is finished, so start over.
|
||||
return Recovery{RetryMode: in.RetryMode, ResumeResults: []byte("{}")}
|
||||
}
|
||||
rec.RetryMode = string(RetryKeepWorktree)
|
||||
return rec
|
||||
}
|
||||
Reference in new issue
Block a user