fix(taskstate): retry transient parks with backoff, escalate when exhausted (#1804)
This commit is contained in:
7 files changed
+189
-3
No files matched your search
@@ -493,6 +493,57 @@ func (d *Daemon) Cycle(ctx context.Context) {
|
||||
d.markPass("cycle")
|
||||
d.dispatchBindings(ctx)
|
||||
d.drainReactions(ctx)
|
||||
d.retryTransientParks(ctx)
|
||||
}
|
||||
|
||||
// retryTransientParks requeues tasks parked for an environmental cause once
|
||||
// their backoff has passed. The store parks a task needs_decision, which
|
||||
// raises the alert, when its retries run out, so this never loops on one.
|
||||
func (d *Daemon) retryTransientParks(ctx context.Context) {
|
||||
if d.Store == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
var after storecontract.TaskCursor
|
||||
for {
|
||||
page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200})
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err)
|
||||
return
|
||||
}
|
||||
for _, t := range page {
|
||||
if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) {
|
||||
d.retryTransientPark(ctx, t.ID)
|
||||
}
|
||||
}
|
||||
if len(page) < 200 {
|
||||
return
|
||||
}
|
||||
last := page[len(page)-1]
|
||||
after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID}
|
||||
}
|
||||
}
|
||||
|
||||
// retryTransientPark requeues one task on its stored retry mode, which the
|
||||
// list projection omits: a remediation must keep continuing its branch.
|
||||
func (d *Daemon) retryTransientPark(ctx context.Context, id int64) {
|
||||
t, err := d.Store.TaskByID(ctx, id)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err)
|
||||
return
|
||||
}
|
||||
if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil {
|
||||
if !errors.Is(err, storecontract.ErrStaleTransition) {
|
||||
d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
d.emit(ctx, events.Event{
|
||||
Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow,
|
||||
Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber,
|
||||
Detail: "retried automatically after a transient park",
|
||||
Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason},
|
||||
})
|
||||
}
|
||||
|
||||
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
type retryStore struct {
|
||||
storecontract.TaskStore
|
||||
parked []workflow.Task
|
||||
retried []int64
|
||||
}
|
||||
|
||||
func (s *retryStore) TasksPage(context.Context, storecontract.TaskPage) ([]workflow.Task, error) {
|
||||
return s.parked, nil
|
||||
}
|
||||
|
||||
func (s *retryStore) TaskByID(_ context.Context, id int64) (*workflow.Task, error) {
|
||||
return &workflow.Task{ID: id}, nil
|
||||
}
|
||||
|
||||
func (s *retryStore) RetryTask(_ context.Context, id int64, _, _, _ string, _ workflowtask.Resume, _ bool) error {
|
||||
s.retried = append(s.retried, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRetryTransientParks(t *testing.T) {
|
||||
old := time.Now().Add(-24 * time.Hour)
|
||||
store := &retryStore{parked: []workflow.Task{
|
||||
{ID: 1, ParkClass: taskstate.ParkTransient, UpdatedAt: old},
|
||||
{ID: 2, ParkClass: taskstate.ParkTransient, UpdatedAt: time.Now()},
|
||||
{ID: 3, ParkClass: taskstate.ParkNeedsDecision, UpdatedAt: old},
|
||||
{ID: 4, ParkClass: taskstate.ParkTransient, RetryCount: taskstate.MaxTransientRetries, UpdatedAt: old},
|
||||
}}
|
||||
d := &Daemon{Store: store, Log: slog.New(slog.DiscardHandler)}
|
||||
d.retryTransientParks(t.Context())
|
||||
if len(store.retried) != 1 || store.retried[0] != 1 {
|
||||
t.Fatalf("retried = %v, want only task 1 (due, transient, retries left)", store.retried)
|
||||
}
|
||||
}
|
||||
@@ -1367,6 +1367,19 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error
|
||||
return i, err
|
||||
}
|
||||
|
||||
const taskRetryCount = `-- name: TaskRetryCount :one
|
||||
SELECT retry_count FROM tasks WHERE id = $1
|
||||
`
|
||||
|
||||
// Read under the lock the park's guard takes, so the escalation decision
|
||||
// cannot interleave with a retry.
|
||||
func (q *Queries) TaskRetryCount(ctx context.Context, id int64) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, taskRetryCount, id)
|
||||
var retry_count int64
|
||||
err := row.Scan(&retry_count)
|
||||
return retry_count, err
|
||||
}
|
||||
|
||||
const transitionTask = `-- name: TransitionTask :execrows
|
||||
UPDATE tasks
|
||||
SET status = $2,
|
||||
|
||||
@@ -150,6 +150,11 @@ WHERE id = @id;
|
||||
-- response write.
|
||||
SELECT rereview_rounds FROM tasks WHERE id = $1;
|
||||
|
||||
-- name: TaskRetryCount :one
|
||||
-- Read under the lock the park's guard takes, so the escalation decision
|
||||
-- cannot interleave with a retry.
|
||||
SELECT retry_count FROM tasks WHERE id = $1;
|
||||
|
||||
-- name: LockTaskStatus :one
|
||||
-- Locks the task's row and returns its current status, so the staleness and
|
||||
-- transition-table checks that decide a guarded write hold against a
|
||||
|
||||
@@ -344,9 +344,13 @@ func (s *Store) ParkTask(ctx context.Context, taskID int64, from, detail, class,
|
||||
if err := guardTransition(ctx, q, taskID, from, workflow.StatusParked); err != nil {
|
||||
return err
|
||||
}
|
||||
retries, err := q.TaskRetryCount(ctx, taskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := q.ParkTask(ctx, postgresdb.ParkTaskParams{
|
||||
ID: taskID, ParkReason: clip(detail, 4000),
|
||||
ParkClass: taskstate.NormalizeParkClass(class), Status: from, ParkBlocker: blocker,
|
||||
ParkClass: taskstate.ParkClassFor(class, int(retries)), Status: from, ParkBlocker: blocker,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package taskstate
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTransientParkRetry(t *testing.T) {
|
||||
parked := time.Date(2026, 11, 8, 0, 0, 0, 0, time.UTC)
|
||||
tests := []struct {
|
||||
name string
|
||||
class string
|
||||
retries int
|
||||
wait time.Duration
|
||||
wantDue bool
|
||||
wantClass ParkClass
|
||||
}{
|
||||
{"fresh park waits out the backoff", ParkTransient, 0, 10 * time.Second, false, ParkTransient},
|
||||
{"first retry after the base wait", ParkTransient, 0, 30 * time.Second, true, ParkTransient},
|
||||
{"backoff doubles", ParkTransient, 2, 119 * time.Second, false, ParkTransient},
|
||||
{"backoff doubles, due", ParkTransient, 2, 120 * time.Second, true, ParkTransient},
|
||||
{"last allowed retry", ParkTransient, MaxTransientRetries - 1, 24 * time.Hour, true, ParkTransient},
|
||||
{"exhausted park escalates and is never retried", ParkTransient, MaxTransientRetries, 24 * time.Hour, false, ParkNeedsDecision},
|
||||
{"needs_decision is never retried", ParkNeedsDecision, 0, 24 * time.Hour, false, ParkNeedsDecision},
|
||||
{"terminal is never retried or escalated", ParkTerminal, MaxTransientRetries, 24 * time.Hour, false, ParkTerminal},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := TransientRetryDue(tt.class, tt.retries, parked, parked.Add(tt.wait)); got != tt.wantDue {
|
||||
t.Errorf("TransientRetryDue = %v, want %v", got, tt.wantDue)
|
||||
}
|
||||
if got := ParkClassFor(tt.class, tt.retries); got != tt.wantClass {
|
||||
t.Errorf("ParkClassFor = %q, want %q", got, tt.wantClass)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package taskstate
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Action is an operator control whose availability is determined solely by a
|
||||
@@ -264,8 +265,8 @@ const (
|
||||
ParkNeedsDecision ParkClass = "needs_decision"
|
||||
// ParkTransient parks are environmental: storage, container pool,
|
||||
// grants, transport. Nothing about the work is wrong; a requeue can
|
||||
// succeed. Nothing retries them automatically yet -- the class is
|
||||
// recorded so a future retry pass has a safe, queryable input.
|
||||
// succeed. The daemon requeues them with backoff (TransientRetryDue);
|
||||
// past MaxTransientRetries the park becomes needs_decision (ParkClassFor).
|
||||
ParkTransient ParkClass = "transient"
|
||||
// ParkTerminal parks cannot be retried into success: the repo left the
|
||||
// config, the owning identity retired. An operator archives these.
|
||||
@@ -316,3 +317,31 @@ func NormalizeParkClass(class string) ParkClass {
|
||||
return ParkNeedsDecision
|
||||
}
|
||||
}
|
||||
|
||||
// MaxTransientRetries is how many times a transient park is requeued
|
||||
// automatically before it needs an operator.
|
||||
const MaxTransientRetries = 5
|
||||
|
||||
// transientBackoffBase is the wait before the first automatic retry; each
|
||||
// further retry waits twice as long.
|
||||
const transientBackoffBase = 30 * time.Second
|
||||
|
||||
// ParkClassFor is the class a park is stored with: NormalizeParkClass, except
|
||||
// a transient park whose task has already used its retries becomes
|
||||
// needs_decision, so it raises the operator alert instead of waiting unseen.
|
||||
func ParkClassFor(class string, retries int) ParkClass {
|
||||
class = NormalizeParkClass(class)
|
||||
if class == ParkTransient && retries >= MaxTransientRetries {
|
||||
return ParkNeedsDecision
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// TransientRetryDue reports whether a task parked as class at parkedAt, after
|
||||
// retries earlier retries, should be requeued automatically at now.
|
||||
func TransientRetryDue(class string, retries int, parkedAt, now time.Time) bool {
|
||||
if class != ParkTransient || retries >= MaxTransientRetries {
|
||||
return false
|
||||
}
|
||||
return !now.Before(parkedAt.Add(transientBackoffBase << retries))
|
||||
}
|
||||
Reference in new issue
Block a user