fix(daemon): park pr_open tasks on permanent forge resolution failures (#1975)

This commit is contained in:
sam committed 2026-11-14 20:09:19 +11:00
1 parent a8d1de8b5e
commit a262151cb9
4 files changed
+433 -18

No files matched your search

+362 -6
View File
@@ -1,38 +1,103 @@
package daemon
import (
"bytes"
"context"
"errors"
"log/slog"
"slices"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
"github.com/samcharles93/archie-core/internal/taskstate"
"github.com/samcharles93/archie-core/internal/worktree"
)
type reconcileStore struct {
storecontract.TaskStore
open []workflowtask.Task
transitions []string
type parkCall struct {
taskID int64
from string
detail string
class taskstate.ParkClass
blocker taskstate.Blocker
}
func (s *reconcileStore) OpenPRs(context.Context) ([]workflowtask.Task, error) { return s.open, nil }
type reconcileStore struct {
storecontract.TaskStore
open []workflowtask.Task
transitions []string
parks []parkCall
parkErr error
reviewCursor int64
commentCursor int64
}
func (s *reconcileStore) OpenPRs(context.Context) ([]workflowtask.Task, error) {
var out []workflowtask.Task
for _, t := range s.open {
if t.Status == workflow.StatusPROpen {
out = append(out, t)
}
}
return out, nil
}
func (s *reconcileStore) Transition(_ context.Context, _ int64, _, to, _ string) error {
s.transitions = append(s.transitions, to)
return nil
}
func (s *reconcileStore) ParkTask(_ context.Context, taskID int64, from, detail, class, blocker string) error {
if s.parkErr != nil {
return s.parkErr
}
if !taskstate.CanTransition(from, workflow.StatusParked) {
return storecontract.ErrIllegalTransition
}
for i := range s.open {
if s.open[i].ID != taskID {
continue
}
if s.open[i].Status != from {
return storecontract.ErrStaleTransition
}
s.open[i].Status = workflow.StatusParked
s.open[i].ParkReason = detail
s.open[i].ParkClass = class
s.open[i].ParkBlocker = blocker
s.parks = append(s.parks, parkCall{
taskID: taskID, from: from, detail: detail, class: class, blocker: blocker,
})
return nil
}
return storecontract.ErrStaleTransition
}
func (s *reconcileStore) SetReviewCursors(_ context.Context, taskID, reviewCursor, commentCursor int64) error {
s.reviewCursor = reviewCursor
s.commentCursor = commentCursor
for i := range s.open {
if s.open[i].ID == taskID {
s.open[i].ReviewCursor = reviewCursor
s.open[i].WatchCommentID = commentCursor
}
}
return nil
}
// identityGatedHub refuses every task the way the execution-identity gate
// refuses an operator-started one.
type identityGatedHub struct {
repoHub
log *slog.Logger
log *slog.Logger
repoForge forge.Forge
repoErr error
}
func (h identityGatedHub) forgeFor(context.Context, *workflow.Task) forge.Forge {
@@ -43,6 +108,13 @@ func (identityGatedHub) treesFor(*workflow.Task) *worktree.Manager {
return &worktree.Manager{WorkDir: "/nonexistent"}
}
func (h identityGatedHub) Repository(context.Context, *workflow.Task, string) (RepositoryAccess, error) {
if h.repoErr != nil {
return RepositoryAccess{}, h.repoErr
}
return RepositoryAccess{Forge: h.repoForge}, nil
}
type mergedForge struct{ forge.Forge }
func (mergedForge) PRState(context.Context, string, string, int) (string, error) {
@@ -51,6 +123,27 @@ func (mergedForge) PRState(context.Context, string, string, int) (string, error)
func (mergedForge) CloseIssue(context.Context, string, string, int, string) error { return nil }
type staticBotSource string
func (b staticBotSource) botUserForTask(*workflowtask.Task) string { return string(b) }
type reconcileReviewForge struct {
forge.Forge
fakeReviewForge
listCalls int
}
func (f *reconcileReviewForge) ListReviews(_ context.Context, _, _ string, _ int, sinceID int64) ([]forge.Review, error) {
f.listCalls++
var out []forge.Review
for _, r := range f.reviews {
if r.ID > sinceID {
out = append(out, r)
}
}
return out, nil
}
// TestReconcilePRsIgnoresTaskIdentity guards #1872: a merged PR of an
// operator-started task stayed In review because the reconciler resolved its
// forge through the task's execution identity.
@@ -77,3 +170,266 @@ func TestReconcilePRsIgnoresTaskIdentity(t *testing.T) {
t.Errorf("forge resolved in org %q, want the task's org", resolvedOrg)
}
}
// TestOpenPRForgeResolutionAndParking guards #1975: scanPRReviews resolves
// operator-started pr_open tasks through s.forges instead of forgeFor,
// permanent forge resolution failures park pr_open tasks with ParkNeedsDecision
// and the expected blocker (stopping subsequent retries), and transient
// errors warn and skip without parking.
func TestOpenPRForgeResolutionAndParking(t *testing.T) {
baseTask := func() workflowtask.Task {
return workflowtask.Task{
ID: 11, Owner: "o", Repo: "r", IssueNumber: 5, PRNumber: 9,
Org: "acme", Identity: "operator", Workflow: "implement", Attempt: 2,
Status: workflow.StatusPROpen,
}
}
tests := []struct {
name string
scan bool // true exercises scanPRReviews; false exercises reconcilePRs
task workflowtask.Task
nilForges bool
nilBus bool
forgeErr error
repoErr error
parkErr error
wantParked bool
wantBlocker taskstate.Blocker
wantParkEvent bool
wantSecondCall bool
wantScan bool
wantLog string
forbidLog string
}{
{
name: "reconcilePRs parks connection_missing and stops retrying",
task: baseTask(),
forgeErr: &forge.AccessError{Code: "connection_missing", Message: "The selected connection no longer exists."},
wantParked: true,
wantBlocker: taskstate.BlockerRepository,
wantParkEvent: true,
wantSecondCall: false,
},
{
name: "scanPRReviews parks operation_denied and stops retrying",
scan: true,
task: baseTask(),
forgeErr: &forge.AccessError{Code: "operation_denied", Operation: "clone", Message: "operation denied"},
wantParked: true,
wantBlocker: taskstate.BlockerConnectionPermit,
wantParkEvent: true,
wantSecondCall: false,
},
{
name: "scanPRReviews parks blockedPark pinFailure",
scan: true,
task: baseTask(),
forgeErr: blockedPark{blocker: taskstate.BlockerAgentAssignment, err: pinFailure{class: taskstate.ParkNeedsDecision, err: errors.New("inactive agent")}},
wantParked: true,
wantBlocker: taskstate.BlockerAgentAssignment,
wantParkEvent: true,
wantSecondCall: false,
},
{
name: "reconcilePRs parks non-transient pinFailure without blocker",
task: baseTask(),
nilBus: true,
forgeErr: &pinFailure{class: taskstate.ParkNeedsDecision, err: errors.New("wrong org")},
wantParked: true,
wantBlocker: "",
wantParkEvent: false,
wantSecondCall: false,
},
{
name: "reconcilePRs skips configuration_unavailable without parking",
task: baseTask(),
forgeErr: &forge.AccessError{Code: "configuration_unavailable", Message: "Settings unavailable."},
wantParked: false,
wantParkEvent: false,
wantSecondCall: true,
wantLog: "PR state check skipped",
},
{
name: "scanPRReviews skips generic error without parking",
scan: true,
task: baseTask(),
forgeErr: errors.New("temporary dial error"),
wantParked: false,
wantParkEvent: false,
wantSecondCall: true,
wantLog: "review scan skipped",
},
{
name: "reconcilePRs skips transient pinFailure without parking",
task: baseTask(),
forgeErr: pinFailure{class: taskstate.ParkTransient, err: errors.New("store timeout")},
wantParked: false,
wantParkEvent: false,
wantSecondCall: true,
wantLog: "PR state check skipped",
},
{
name: "scanPRReviews skips transient pinFailure without parking",
scan: true,
task: baseTask(),
forgeErr: pinFailure{class: taskstate.ParkTransient, err: errors.New("store timeout")},
wantParked: false,
wantParkEvent: false,
wantSecondCall: true,
wantLog: "review scan skipped",
},
{
name: "scanPRReviews resolves operator-started task via s.forges",
scan: true,
task: baseTask(),
wantParked: false,
wantParkEvent: false,
wantSecondCall: true,
wantScan: true,
forbidLog: "no forge for task",
},
{
name: "reconcilePRs suppresses parked event on stale transition",
task: baseTask(),
forgeErr: &forge.AccessError{Code: "connection_missing", Message: "missing"},
parkErr: storecontract.ErrStaleTransition,
wantParked: false,
wantParkEvent: false,
wantSecondCall: true,
},
{
name: "reconcilePRs skips task without repository when noop forge returned",
task: workflowtask.Task{ID: 12, PRNumber: 4, Identity: "operator", Status: workflow.StatusPROpen},
wantParked: false,
wantParkEvent: false,
wantSecondCall: false,
wantLog: "PR state check skipped",
},
{
name: "reconcilePRs with nil forges falls back to Repository and preserves typed error",
task: baseTask(),
nilForges: true,
repoErr: &forge.AccessError{Code: "connection_missing", Message: "missing via repo"},
wantParked: true,
wantBlocker: taskstate.BlockerRepository,
wantParkEvent: true,
wantSecondCall: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var logBuf bytes.Buffer
log := slog.New(slog.NewTextHandler(&logBuf, nil))
bus := events.NewBus()
defer bus.Close()
sub := bus.Subscribe(4)
defer sub.Close()
store := &reconcileStore{open: []workflowtask.Task{tt.task}, parkErr: tt.parkErr}
reviewForge := &reconcileReviewForge{
reviews: []forge.Review{{ID: 10, Author: "maint", State: forge.ReviewStateRequestedChanges, Body: "fix"}},
writers: map[string]bool{"maint": true},
}
var published []string
var forgeCalls int
var resolvedOrg org.OrgID
var forgesFn RepositoryForgeResolver
if !tt.nilForges {
forgesFn = func(ctx context.Context, _, _ string) (forge.Forge, error) {
forgeCalls++
resolvedOrg = org.OrgFromContext(ctx)
if tt.forgeErr != nil {
return nil, tt.forgeErr
}
return reviewForge, nil
}
}
var scannerBus *events.Bus
if !tt.nilBus {
scannerBus = bus
}
s := &ReviewScanner{
store: store,
bus: scannerBus,
log: log,
repos: identityGatedHub{log: log, repoErr: tt.repoErr},
bots: staticBotSource("bot"),
forges: forgesFn,
publish: func(_ context.Context, r workintake.ReviewCommentEnvelope) error {
published = append(published, r.Author)
return nil
},
}
run := func() error {
if tt.scan {
return s.scanPRReviews(t.Context())
}
return s.reconcilePRs(t.Context())
}
if err := run(); err != nil {
t.Fatalf("first pass error: %v", err)
}
if err := run(); err != nil {
t.Fatalf("second pass error: %v", err)
}
if tt.wantParked {
if len(store.parks) != 1 {
t.Fatalf("parks = %d, want 1", len(store.parks))
}
got := store.parks[0]
if got.taskID != tt.task.ID || got.from != workflow.StatusPROpen || got.class != taskstate.ParkNeedsDecision || got.blocker != tt.wantBlocker {
t.Fatalf("park = %+v, want task=%d from=%s class=%s blocker=%q", got, tt.task.ID, workflow.StatusPROpen, taskstate.ParkNeedsDecision, tt.wantBlocker)
}
} else if len(store.parks) != 0 {
t.Fatalf("unexpected parks: %+v", store.parks)
}
if !tt.nilForges && tt.task.HasRepository() {
wantCalls := 1
if tt.wantSecondCall {
wantCalls = 2
}
if forgeCalls != wantCalls {
t.Fatalf("forgeCalls = %d, want %d", forgeCalls, wantCalls)
}
}
select {
case ev := <-sub.C:
if !tt.wantParkEvent {
t.Fatalf("unexpected event emitted: %+v", ev)
}
if ev.Kind != events.KindParked || ev.TaskID != tt.task.ID || ev.Attempt != tt.task.Attempt || ev.Data["park_class"] != taskstate.ParkNeedsDecision {
t.Fatalf("park event = %+v, want KindParked task=%d attempt=%d park_class=%s", ev, tt.task.ID, tt.task.Attempt, taskstate.ParkNeedsDecision)
}
default:
if tt.wantParkEvent {
t.Fatal("expected KindParked event, got none")
}
}
if tt.wantScan {
if resolvedOrg != tt.task.Org {
t.Fatalf("resolvedOrg = %q, want %q", resolvedOrg, tt.task.Org)
}
if reviewForge.listCalls == 0 || !slices.Equal(published, []string{"maint"}) || store.reviewCursor != 10 {
t.Fatalf("scan did not execute via s.forges: listCalls=%d published=%v cursor=%d", reviewForge.listCalls, published, store.reviewCursor)
}
}
logs := logBuf.String()
if tt.wantLog != "" && !strings.Contains(logs, tt.wantLog) {
t.Errorf("logs %q missing %q", logs, tt.wantLog)
}
if tt.forbidLog != "" && strings.Contains(logs, tt.forbidLog) {
t.Errorf("logs %q unexpectedly contained %q", logs, tt.forbidLog)
}
})
}
}
+66 -7
View File
@@ -5,7 +5,9 @@ import (
"errors"
"fmt"
"log/slog"
"strings"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
@@ -14,6 +16,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// ReactionPublisher delivers one reaction to the reaction stream. The
@@ -175,8 +178,8 @@ func NewReviewScanner(
return s
}
// scanPRReviews scans every open PR task for review activity with its
// identity's forge. Forges without review reading are skipped.
// scanPRReviews scans every open PR task for review activity through its
// repository forge. Forges without review reading are skipped.
func (s *ReviewScanner) scanPRReviews(ctx context.Context) error {
tasks, err := s.store.OpenPRs(ctx)
if err != nil {
@@ -185,11 +188,15 @@ func (s *ReviewScanner) scanPRReviews(ctx context.Context) error {
var failed []string
for i := range tasks {
t := &tasks[i]
reader, ok := s.repos.forgeFor(ctx, t).(forge.PullRequestReviewReader)
fg, ok := s.openPRForge(ctx, t, "review scan skipped: no forge for task")
if !ok {
continue
}
reader, ok := fg.(forge.PullRequestReviewReader)
if !ok {
continue // the noop forge and any forge without the capability
}
checker, ok := s.repos.forgeFor(ctx, t).(forge.WriteAccessChecker)
checker, ok := fg.(forge.WriteAccessChecker)
if !ok {
s.log.WarnContext(ctx, "review scan skipped: forge cannot verify write access", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
continue
@@ -270,9 +277,8 @@ func (s *ReviewScanner) reconcilePRs(ctx context.Context) error {
}
var failed []string
for _, t := range tasks {
fg, err := s.reconcileForge(ctx, &t)
if err != nil {
s.log.WarnContext(ctx, "PR state check skipped: no forge for task", "task", t.ID, "pr", t.PRNumber, "err", err)
fg, ok := s.openPRForge(ctx, &t, "PR state check skipped: no forge for task")
if !ok {
continue
}
trees := s.repos.treesFor(&t)
@@ -339,9 +345,62 @@ func (s *ReviewScanner) reconcileForge(ctx context.Context, task *workflow.Task)
}
return fg, nil
}
if s.forges == nil {
resolved, err := s.repos.Repository(ctx, task, config.OperationClone)
if err != nil {
return nil, err
}
return resolved.Forge, nil
}
return s.forges(org.WithOrg(ctx, org.OrgID(task.OrgOrDefault())), task.Owner, task.Repo)
}
// permanentForgeFailure reports whether err from reconcileForge is permanent
// and names the park blocker if one applies.
func permanentForgeFailure(err error) (taskstate.Blocker, bool) {
if blocker := parkBlocker(err); blocker != "" {
return blocker, true
}
if failure, ok := errors.AsType[pinFailure](err); ok {
return "", failure.class != taskstate.ParkTransient
}
if failure, ok := errors.AsType[*pinFailure](err); ok && failure != nil {
return "", failure.class != taskstate.ParkTransient
}
return "", false
}
// openPRForge resolves the forge for an open PR task. Transient errors warn
// and skip; permanent errors park the task out of pr_open so it stops retrying
// on subsequent ticks.
func (s *ReviewScanner) openPRForge(ctx context.Context, t *workflow.Task, skipMsg string) (forge.Forge, bool) {
fg, err := s.reconcileForge(ctx, t)
if err == nil {
return fg, true
}
blocker, permanent := permanentForgeFailure(err)
if !permanent {
s.log.WarnContext(ctx, skipMsg, "task", t.ID, "pr", t.PRNumber, "err", err)
return nil, false
}
reason := err.Error()
if err := s.store.ParkTask(ctx, t.ID, workflow.StatusPROpen, reason, taskstate.ParkNeedsDecision, blocker); err != nil {
if !errors.Is(err, storecontract.ErrStaleTransition) {
s.log.WarnContext(ctx, "park pr_open task failed", "task", t.ID, "pr", t.PRNumber, "reason", reason, "err", err)
}
return nil, false
}
s.log.WarnContext(ctx, "pr_open task parked: forge resolution failed", "task", t.ID, "pr", t.PRNumber, "blocker", blocker, "err", err)
if s.bus != nil {
s.bus.Publish(ctx, events.Event{
Kind: events.KindParked, TaskID: t.ID, Attempt: t.Attempt,
Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber, Workflow: t.Workflow,
Detail: reason, Data: map[string]any{"park_class": taskstate.ParkNeedsDecision},
})
}
return nil, false
}
// closeResolvedIssue closes a finished task's forge issue, skipping chat
// tasks. Failures are logged.
func (s *ReviewScanner) closeResolvedIssue(ctx context.Context, task *workflow.Task, comment string) {
+4 -4
View File
@@ -340,10 +340,10 @@ func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail s
return tx.Commit(ctx)
}
// ParkTask is the classified park write: the same guarded running->parked
// transition Transition performs, carrying the park class the site chose.
// The park is only legal from running (the transition table), so a stale or
// off-table from is refused before anything is written.
// ParkTask is the classified park write: the same guarded transition to parked
// that Transition performs, carrying the park class and blocker the site chose.
// The move from from to parked is checked against the transition table, so a
// stale or off-table from is refused before anything is written.
func (s *Store) ParkTask(ctx context.Context, taskID int64, from, detail, class, blocker string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
+1 -1
View File
@@ -28,7 +28,7 @@ var executionTransitions = map[string][]string{
Running: {Queued, WaitingApproval, PROpen, Completed, Parked, Declined},
WaitingApproval: {Queued, Declined},
Parked: {Queued, Dead, Declined},
PROpen: {Merged, Rejected, Queued, Declined},
PROpen: {Merged, Rejected, Queued, Parked, Declined},
Merged: nil,
Rejected: nil,