diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index b42353bcc..1fec3ed63 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -493,6 +493,57 @@ func (d *Daemon) Cycle(ctx context.Context) { d.markPass("cycle") d.dispatchBindings(ctx) d.drainReactions(ctx) + d.retryTransientParks(ctx) +} + +// retryTransientParks requeues tasks parked for an environmental cause once +// their backoff has passed. The store parks a task needs_decision, which +// raises the alert, when its retries run out, so this never loops on one. +func (d *Daemon) retryTransientParks(ctx context.Context) { + if d.Store == nil { + return + } + now := time.Now() + var after storecontract.TaskCursor + for { + page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200}) + if err != nil { + d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err) + return + } + for _, t := range page { + if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) { + d.retryTransientPark(ctx, t.ID) + } + } + if len(page) < 200 { + return + } + last := page[len(page)-1] + after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID} + } +} + +// retryTransientPark requeues one task on its stored retry mode, which the +// list projection omits: a remediation must keep continuing its branch. +func (d *Daemon) retryTransientPark(ctx context.Context, id int64) { + t, err := d.Store.TaskByID(ctx, id) + if err != nil { + d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err) + return + } + if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil { + if !errors.Is(err, storecontract.ErrStaleTransition) { + d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err) + } + return + } + d.emit(ctx, events.Event{ + Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow, + Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber, + Detail: "retried automatically after a transient park", + Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason}, + }) } func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error { diff --git a/internal/daemon/transient_retry_test.go b/internal/daemon/transient_retry_test.go new file mode 100644 index 000000000..a6db2ded8 --- /dev/null +++ b/internal/daemon/transient_retry_test.go @@ -0,0 +1,47 @@ +package daemon + +import ( + "context" + "log/slog" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/workflow" + workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +type retryStore struct { + storecontract.TaskStore + parked []workflow.Task + retried []int64 +} + +func (s *retryStore) TasksPage(context.Context, storecontract.TaskPage) ([]workflow.Task, error) { + return s.parked, nil +} + +func (s *retryStore) TaskByID(_ context.Context, id int64) (*workflow.Task, error) { + return &workflow.Task{ID: id}, nil +} + +func (s *retryStore) RetryTask(_ context.Context, id int64, _, _, _ string, _ workflowtask.Resume, _ bool) error { + s.retried = append(s.retried, id) + return nil +} + +func TestRetryTransientParks(t *testing.T) { + old := time.Now().Add(-24 * time.Hour) + store := &retryStore{parked: []workflow.Task{ + {ID: 1, ParkClass: taskstate.ParkTransient, UpdatedAt: old}, + {ID: 2, ParkClass: taskstate.ParkTransient, UpdatedAt: time.Now()}, + {ID: 3, ParkClass: taskstate.ParkNeedsDecision, UpdatedAt: old}, + {ID: 4, ParkClass: taskstate.ParkTransient, RetryCount: taskstate.MaxTransientRetries, UpdatedAt: old}, + }} + d := &Daemon{Store: store, Log: slog.New(slog.DiscardHandler)} + d.retryTransientParks(t.Context()) + if len(store.retried) != 1 || store.retried[0] != 1 { + t.Fatalf("retried = %v, want only task 1 (due, transient, retries left)", store.retried) + } +} diff --git a/internal/infrastructure/postgres/postgresdb/tasks.sql.go b/internal/infrastructure/postgres/postgresdb/tasks.sql.go index 70e77e2db..2cc95a2c3 100644 --- a/internal/infrastructure/postgres/postgresdb/tasks.sql.go +++ b/internal/infrastructure/postgres/postgresdb/tasks.sql.go @@ -1367,6 +1367,19 @@ func (q *Queries) TaskByPR(ctx context.Context, arg TaskByPRParams) (Task, error return i, err } +const taskRetryCount = `-- name: TaskRetryCount :one +SELECT retry_count FROM tasks WHERE id = $1 +` + +// Read under the lock the park's guard takes, so the escalation decision +// cannot interleave with a retry. +func (q *Queries) TaskRetryCount(ctx context.Context, id int64) (int64, error) { + row := q.db.QueryRow(ctx, taskRetryCount, id) + var retry_count int64 + err := row.Scan(&retry_count) + return retry_count, err +} + const transitionTask = `-- name: TransitionTask :execrows UPDATE tasks SET status = $2, diff --git a/internal/infrastructure/postgres/queries/tasks.sql b/internal/infrastructure/postgres/queries/tasks.sql index 54f4a7f53..8a6c38583 100644 --- a/internal/infrastructure/postgres/queries/tasks.sql +++ b/internal/infrastructure/postgres/queries/tasks.sql @@ -150,6 +150,11 @@ WHERE id = @id; -- response write. SELECT rereview_rounds FROM tasks WHERE id = $1; +-- name: TaskRetryCount :one +-- Read under the lock the park's guard takes, so the escalation decision +-- cannot interleave with a retry. +SELECT retry_count FROM tasks WHERE id = $1; + -- name: LockTaskStatus :one -- Locks the task's row and returns its current status, so the staleness and -- transition-table checks that decide a guarded write hold against a diff --git a/internal/infrastructure/postgres/store.go b/internal/infrastructure/postgres/store.go index 94e3906d5..7487e9375 100644 --- a/internal/infrastructure/postgres/store.go +++ b/internal/infrastructure/postgres/store.go @@ -344,9 +344,13 @@ func (s *Store) ParkTask(ctx context.Context, taskID int64, from, detail, class, if err := guardTransition(ctx, q, taskID, from, workflow.StatusParked); err != nil { return err } + retries, err := q.TaskRetryCount(ctx, taskID) + if err != nil { + return err + } n, err := q.ParkTask(ctx, postgresdb.ParkTaskParams{ ID: taskID, ParkReason: clip(detail, 4000), - ParkClass: taskstate.NormalizeParkClass(class), Status: from, ParkBlocker: blocker, + ParkClass: taskstate.ParkClassFor(class, int(retries)), Status: from, ParkBlocker: blocker, }) if err != nil { return err diff --git a/internal/taskstate/park_retry_test.go b/internal/taskstate/park_retry_test.go new file mode 100644 index 000000000..7f301eccf --- /dev/null +++ b/internal/taskstate/park_retry_test.go @@ -0,0 +1,37 @@ +package taskstate + +import ( + "testing" + "time" +) + +func TestTransientParkRetry(t *testing.T) { + parked := time.Date(2026, 11, 8, 0, 0, 0, 0, time.UTC) + tests := []struct { + name string + class string + retries int + wait time.Duration + wantDue bool + wantClass ParkClass + }{ + {"fresh park waits out the backoff", ParkTransient, 0, 10 * time.Second, false, ParkTransient}, + {"first retry after the base wait", ParkTransient, 0, 30 * time.Second, true, ParkTransient}, + {"backoff doubles", ParkTransient, 2, 119 * time.Second, false, ParkTransient}, + {"backoff doubles, due", ParkTransient, 2, 120 * time.Second, true, ParkTransient}, + {"last allowed retry", ParkTransient, MaxTransientRetries - 1, 24 * time.Hour, true, ParkTransient}, + {"exhausted park escalates and is never retried", ParkTransient, MaxTransientRetries, 24 * time.Hour, false, ParkNeedsDecision}, + {"needs_decision is never retried", ParkNeedsDecision, 0, 24 * time.Hour, false, ParkNeedsDecision}, + {"terminal is never retried or escalated", ParkTerminal, MaxTransientRetries, 24 * time.Hour, false, ParkTerminal}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := TransientRetryDue(tt.class, tt.retries, parked, parked.Add(tt.wait)); got != tt.wantDue { + t.Errorf("TransientRetryDue = %v, want %v", got, tt.wantDue) + } + if got := ParkClassFor(tt.class, tt.retries); got != tt.wantClass { + t.Errorf("ParkClassFor = %q, want %q", got, tt.wantClass) + } + }) + } +} diff --git a/internal/taskstate/taskstate.go b/internal/taskstate/taskstate.go index ef73fef68..575d61ff3 100644 --- a/internal/taskstate/taskstate.go +++ b/internal/taskstate/taskstate.go @@ -5,6 +5,7 @@ package taskstate import ( "fmt" "slices" + "time" ) // Action is an operator control whose availability is determined solely by a @@ -264,8 +265,8 @@ const ( ParkNeedsDecision ParkClass = "needs_decision" // ParkTransient parks are environmental: storage, container pool, // grants, transport. Nothing about the work is wrong; a requeue can - // succeed. Nothing retries them automatically yet -- the class is - // recorded so a future retry pass has a safe, queryable input. + // succeed. The daemon requeues them with backoff (TransientRetryDue); + // past MaxTransientRetries the park becomes needs_decision (ParkClassFor). ParkTransient ParkClass = "transient" // ParkTerminal parks cannot be retried into success: the repo left the // config, the owning identity retired. An operator archives these. @@ -316,3 +317,31 @@ func NormalizeParkClass(class string) ParkClass { return ParkNeedsDecision } } + +// MaxTransientRetries is how many times a transient park is requeued +// automatically before it needs an operator. +const MaxTransientRetries = 5 + +// transientBackoffBase is the wait before the first automatic retry; each +// further retry waits twice as long. +const transientBackoffBase = 30 * time.Second + +// ParkClassFor is the class a park is stored with: NormalizeParkClass, except +// a transient park whose task has already used its retries becomes +// needs_decision, so it raises the operator alert instead of waiting unseen. +func ParkClassFor(class string, retries int) ParkClass { + class = NormalizeParkClass(class) + if class == ParkTransient && retries >= MaxTransientRetries { + return ParkNeedsDecision + } + return class +} + +// TransientRetryDue reports whether a task parked as class at parkedAt, after +// retries earlier retries, should be requeued automatically at now. +func TransientRetryDue(class string, retries int, parkedAt, now time.Time) bool { + if class != ParkTransient || retries >= MaxTransientRetries { + return false + } + return !now.Before(parkedAt.Add(transientBackoffBase << retries)) +}