fix(taskstate): retry transient parks with backoff, escalate when exhausted (#1804)
deploy / plan (push) Successful in 16s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s
deploy / gate (push) Canceled after 2m21s

This commit is contained in:
sam committed 2026-11-08 17:19:14 +11:00
1 parent af7577625f
commit 860b2c85e0
7 files changed
+189 -3

No files matched your search

+51
View File
@@ -493,6 +493,57 @@ func (d *Daemon) Cycle(ctx context.Context) {
d.markPass("cycle")
d.dispatchBindings(ctx)
d.drainReactions(ctx)
d.retryTransientParks(ctx)
}
// retryTransientParks requeues tasks parked for an environmental cause once
// their backoff has passed. The store parks a task needs_decision, which
// raises the alert, when its retries run out, so this never loops on one.
func (d *Daemon) retryTransientParks(ctx context.Context) {
if d.Store == nil {
return
}
now := time.Now()
var after storecontract.TaskCursor
for {
page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200})
if err != nil {
d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err)
return
}
for _, t := range page {
if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) {
d.retryTransientPark(ctx, t.ID)
}
}
if len(page) < 200 {
return
}
last := page[len(page)-1]
after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID}
}
}
// retryTransientPark requeues one task on its stored retry mode, which the
// list projection omits: a remediation must keep continuing its branch.
func (d *Daemon) retryTransientPark(ctx context.Context, id int64) {
t, err := d.Store.TaskByID(ctx, id)
if err != nil {
d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err)
return
}
if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil {
if !errors.Is(err, storecontract.ErrStaleTransition) {
d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err)
}
return
}
d.emit(ctx, events.Event{
Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow,
Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber,
Detail: "retried automatically after a transient park",
Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason},
})
}
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
+47
View File
@@ -0,0 +1,47 @@
package daemon
import (
"context"
"log/slog"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/taskstate"
)
type retryStore struct {
storecontract.TaskStore
parked []workflow.Task
retried []int64
}
func (s *retryStore) TasksPage(context.Context, storecontract.TaskPage) ([]workflow.Task, error) {
return s.parked, nil
}
func (s *retryStore) TaskByID(_ context.Context, id int64) (*workflow.Task, error) {
return &workflow.Task{ID: id}, nil
}
func (s *retryStore) RetryTask(_ context.Context, id int64, _, _, _ string, _ workflowtask.Resume, _ bool) error {
s.retried = append(s.retried, id)
return nil
}
func TestRetryTransientParks(t *testing.T) {
old := time.Now().Add(-24 * time.Hour)
store := &retryStore{parked: []workflow.Task{
{ID: 1, ParkClass: taskstate.ParkTransient, UpdatedAt: old},
{ID: 2, ParkClass: taskstate.ParkTransient, UpdatedAt: time.Now()},
{ID: 3, ParkClass: taskstate.ParkNeedsDecision, UpdatedAt: old},
{ID: 4, ParkClass: taskstate.ParkTransient, RetryCount: taskstate.MaxTransientRetries, UpdatedAt: old},
}}
d := &Daemon{Store: store, Log: slog.New(slog.DiscardHandler)}
d.retryTransientParks(t.Context())
if len(store.retried) != 1 || store.retried[0] != 1 {
t.Fatalf("retried = %v, want only task 1 (due, transient, retries left)", store.retried)
}
}
@@ -1367,6 +1367,19 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error
return i, err
}
const taskRetryCount = `-- name: TaskRetryCount :one
SELECT retry_count FROM tasks WHERE id = $1
`
// Read under the lock the park's guard takes, so the escalation decision
// cannot interleave with a retry.
func (q *Queries) TaskRetryCount(ctx context.Context, id int64) (int64, error) {
row := q.db.QueryRow(ctx, taskRetryCount, id)
var retry_count int64
err := row.Scan(&retry_count)
return retry_count, err
}
const transitionTask = `-- name: TransitionTask :execrows
UPDATE tasks
SET status = $2,
@@ -150,6 +150,11 @@ WHERE id = @id;
-- response write.
SELECT rereview_rounds FROM tasks WHERE id = $1;
-- name: TaskRetryCount :one
-- Read under the lock the park's guard takes, so the escalation decision
-- cannot interleave with a retry.
SELECT retry_count FROM tasks WHERE id = $1;
-- name: LockTaskStatus :one
-- Locks the task's row and returns its current status, so the staleness and
-- transition-table checks that decide a guarded write hold against a
+5 -1
View File
@@ -344,9 +344,13 @@ func (s *Store) ParkTask(ctx context.Context, taskID int64, from, detail, class,
if err := guardTransition(ctx, q, taskID, from, workflow.StatusParked); err != nil {
return err
}
retries, err := q.TaskRetryCount(ctx, taskID)
if err != nil {
return err
}
n, err := q.ParkTask(ctx, postgresdb.ParkTaskParams{
ID: taskID, ParkReason: clip(detail, 4000),
ParkClass: taskstate.NormalizeParkClass(class), Status: from, ParkBlocker: blocker,
ParkClass: taskstate.ParkClassFor(class, int(retries)), Status: from, ParkBlocker: blocker,
})
if err != nil {
return err
+37
View File
@@ -0,0 +1,37 @@
package taskstate
import (
"testing"
"time"
)
func TestTransientParkRetry(t *testing.T) {
parked := time.Date(2026, 11, 8, 0, 0, 0, 0, time.UTC)
tests := []struct {
name string
class string
retries int
wait time.Duration
wantDue bool
wantClass ParkClass
}{
{"fresh park waits out the backoff", ParkTransient, 0, 10 * time.Second, false, ParkTransient},
{"first retry after the base wait", ParkTransient, 0, 30 * time.Second, true, ParkTransient},
{"backoff doubles", ParkTransient, 2, 119 * time.Second, false, ParkTransient},
{"backoff doubles, due", ParkTransient, 2, 120 * time.Second, true, ParkTransient},
{"last allowed retry", ParkTransient, MaxTransientRetries - 1, 24 * time.Hour, true, ParkTransient},
{"exhausted park escalates and is never retried", ParkTransient, MaxTransientRetries, 24 * time.Hour, false, ParkNeedsDecision},
{"needs_decision is never retried", ParkNeedsDecision, 0, 24 * time.Hour, false, ParkNeedsDecision},
{"terminal is never retried or escalated", ParkTerminal, MaxTransientRetries, 24 * time.Hour, false, ParkTerminal},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := TransientRetryDue(tt.class, tt.retries, parked, parked.Add(tt.wait)); got != tt.wantDue {
t.Errorf("TransientRetryDue = %v, want %v", got, tt.wantDue)
}
if got := ParkClassFor(tt.class, tt.retries); got != tt.wantClass {
t.Errorf("ParkClassFor = %q, want %q", got, tt.wantClass)
}
})
}
}
+31 -2
View File
@@ -5,6 +5,7 @@ package taskstate
import (
"fmt"
"slices"
"time"
)
// Action is an operator control whose availability is determined solely by a
@@ -264,8 +265,8 @@ const (
ParkNeedsDecision ParkClass = "needs_decision"
// ParkTransient parks are environmental: storage, container pool,
// grants, transport. Nothing about the work is wrong; a requeue can
// succeed. Nothing retries them automatically yet -- the class is
// recorded so a future retry pass has a safe, queryable input.
// succeed. The daemon requeues them with backoff (TransientRetryDue);
// past MaxTransientRetries the park becomes needs_decision (ParkClassFor).
ParkTransient ParkClass = "transient"
// ParkTerminal parks cannot be retried into success: the repo left the
// config, the owning identity retired. An operator archives these.
@@ -316,3 +317,31 @@ func NormalizeParkClass(class string) ParkClass {
return ParkNeedsDecision
}
}
// MaxTransientRetries is how many times a transient park is requeued
// automatically before it needs an operator.
const MaxTransientRetries = 5
// transientBackoffBase is the wait before the first automatic retry; each
// further retry waits twice as long.
const transientBackoffBase = 30 * time.Second
// ParkClassFor is the class a park is stored with: NormalizeParkClass, except
// a transient park whose task has already used its retries becomes
// needs_decision, so it raises the operator alert instead of waiting unseen.
func ParkClassFor(class string, retries int) ParkClass {
class = NormalizeParkClass(class)
if class == ParkTransient && retries >= MaxTransientRetries {
return ParkNeedsDecision
}
return class
}
// TransientRetryDue reports whether a task parked as class at parkedAt, after
// retries earlier retries, should be requeued automatically at now.
func TransientRetryDue(class string, retries int, parkedAt, now time.Time) bool {
if class != ParkTransient || retries >= MaxTransientRetries {
return false
}
return !now.Before(parkedAt.Add(transientBackoffBase << retries))
}