Implement max_retries cap on maybeRetryParked to prevent infinite parked-task loops (archie)
Implements #61
This commit is contained in:
1 parent
7534cbe2c4
commit
d2b2f29096
11 files changed
+402
-19
No files matched your search
@@ -7,6 +7,7 @@ poll_interval = "60s"
|
||||
label = "archie"
|
||||
bot_user = "vectis-labs-archie"
|
||||
diff_cap_lines = 400
|
||||
# max_retries = 3 # how many times a parked task is retried before permanent park (global default; set per-repo to override)
|
||||
|
||||
[forge]
|
||||
type = "github"
|
||||
@@ -23,6 +24,7 @@ working = "archie:working"
|
||||
waiting = "archie:waiting"
|
||||
pr = "archie:pr"
|
||||
parked = "archie:parked"
|
||||
# dead = "archie:dead" # applied when a task exhausts max_retries
|
||||
|
||||
[models] # role -> ai-sdk runtime ref (used by agent workflows, Phase 4b)
|
||||
triage = "openai/gpt-5.4"
|
||||
|
||||
@@ -62,6 +62,10 @@ type Repo struct {
|
||||
// survives task completion and retains data across tasks. Use for
|
||||
// repos with expensive build artifacts or large dependency trees.
|
||||
PersistentStorage bool `toml:"persistent_storage"`
|
||||
// MaxRetries caps how many times a parked task is retried before
|
||||
// being permanently parked (status "dead"). 0 means use the
|
||||
// global Config.MaxRetries.
|
||||
MaxRetries int `toml:"max_retries"`
|
||||
}
|
||||
|
||||
// Protected reports whether path matches a protected suffix.
|
||||
@@ -118,6 +122,15 @@ func (r Repo) BaseBranch() string {
|
||||
return r.Base
|
||||
}
|
||||
|
||||
// EffectiveMaxRetries returns the per-repo override when set (>0),
|
||||
// otherwise the global default.
|
||||
func (r Repo) EffectiveMaxRetries(global int) int {
|
||||
if r.MaxRetries > 0 {
|
||||
return r.MaxRetries
|
||||
}
|
||||
return global
|
||||
}
|
||||
|
||||
// Budgets bound every agent stage. Zero disables a limit.
|
||||
type Budgets struct {
|
||||
MaxSteps int `toml:"max_steps"`
|
||||
@@ -176,6 +189,7 @@ var dispatchLabelDefaults = map[string]string{
|
||||
"waiting": "archie:waiting",
|
||||
"pr": "archie:pr",
|
||||
"parked": "archie:parked",
|
||||
"dead": "archie:dead",
|
||||
}
|
||||
|
||||
// StateLabel returns the configured label for a state name. Falls back
|
||||
@@ -196,7 +210,7 @@ func (d Dispatch) StateLabel(state string) string {
|
||||
func (d Dispatch) LabelValues() []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, k := range []string{"queued", "working", "waiting", "pr", "parked"} {
|
||||
for _, k := range []string{"queued", "working", "waiting", "pr", "parked", "dead"} {
|
||||
v := d.StateLabel(k)
|
||||
if v != "" && !seen[v] {
|
||||
seen[v] = true
|
||||
@@ -215,6 +229,9 @@ type Config struct {
|
||||
// (plugin-defined workflows override built-ins). When empty,
|
||||
// only built-in workflows are available.
|
||||
SkillsDir string `toml:"skills_dir"`
|
||||
// MaxRetries caps how many times a parked task is retried before
|
||||
// being permanently parked (status "dead"). Defaults to 3.
|
||||
MaxRetries int `toml:"max_retries"`
|
||||
// PluginDir is an optional path to a directory of Yaegi-interpreted
|
||||
// daemon plugins (*.go files). Each file must export a "Plugin"
|
||||
// variable satisfying the plugin.Plugin interface. Failed plugins
|
||||
@@ -373,6 +390,9 @@ func Load(path string) (Config, error) {
|
||||
cfg.Dispatch.Labels[k] = v
|
||||
}
|
||||
}
|
||||
if cfg.MaxRetries == 0 {
|
||||
cfg.MaxRetries = 3
|
||||
}
|
||||
if cfg.BotUser == "" {
|
||||
return cfg, errors.New("config: bot_user is required")
|
||||
}
|
||||
|
||||
@@ -91,16 +91,16 @@ func TestDispatchCustomLabels(t *testing.T) {
|
||||
t.Errorf("AckReaction: got %q, want empty", d.AckReaction)
|
||||
}
|
||||
|
||||
// LabelValues returns all five state labels.
|
||||
// LabelValues returns all six state labels.
|
||||
vals := d.LabelValues()
|
||||
if len(vals) != 5 {
|
||||
t.Errorf("LabelValues: got %d values, want 5: %v", len(vals), vals)
|
||||
if len(vals) != 6 {
|
||||
t.Errorf("LabelValues: got %d values, want 6: %v", len(vals), vals)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, v := range vals {
|
||||
seen[v] = true
|
||||
}
|
||||
for _, want := range []string{"bot:queued", "bot:working", "bot:parked", "archie:waiting", "archie:pr"} {
|
||||
for _, want := range []string{"bot:queued", "bot:working", "bot:parked", "archie:waiting", "archie:pr", "archie:dead"} {
|
||||
if !seen[want] {
|
||||
t.Errorf("LabelValues missing %q", want)
|
||||
}
|
||||
@@ -118,9 +118,9 @@ func TestDispatchLabelValuesDedup(t *testing.T) {
|
||||
},
|
||||
}
|
||||
vals := d.LabelValues()
|
||||
// Should have "bot:queued" plus fallbacks for waiting and pr.
|
||||
if len(vals) != 3 {
|
||||
t.Errorf("LabelValues: got %d values, want 3: %v", len(vals), vals)
|
||||
// Should have "bot:queued" plus fallbacks for waiting, pr, and dead.
|
||||
if len(vals) != 4 {
|
||||
t.Errorf("LabelValues: got %d values, want 4: %v", len(vals), vals)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -322,8 +322,9 @@ func (d *Daemon) pollEither(ctx context.Context, repo config.Repo) []forge.Issue
|
||||
return out
|
||||
}
|
||||
|
||||
// maybeRetryParked requeues a parked task whose archie:parked label a
|
||||
// human has removed — the forge-native retry trigger.
|
||||
// maybeRetryParked requeues a parked task whose state label a human has
|
||||
// removed — the forge-native retry trigger. When retry_count reaches the
|
||||
// configured max_retries the task moves to dead instead of requeuing.
|
||||
func (d *Daemon) maybeRetryParked(ctx context.Context, repo config.Repo, is forge.Issue) {
|
||||
if hasLabel(is.Labels, d.Cfg.Dispatch.StateLabel("parked")) {
|
||||
return
|
||||
@@ -332,15 +333,50 @@ func (d *Daemon) maybeRetryParked(ctx context.Context, repo config.Repo, is forg
|
||||
if err != nil || task == nil || task.Status != store.StatusParked {
|
||||
return
|
||||
}
|
||||
|
||||
maxRetries := repo.EffectiveMaxRetries(d.Cfg.MaxRetries)
|
||||
|
||||
if task.RetryCount >= maxRetries {
|
||||
d.markDead(ctx, repo, task, maxRetries)
|
||||
return
|
||||
}
|
||||
|
||||
if err := d.Store.Requeue(ctx, task.ID, store.StatusParked, ""); err != nil {
|
||||
d.Log.Error("label-triggered requeue failed", "issue", is.Number, "err", err)
|
||||
return
|
||||
}
|
||||
d.Log.Info("parked task requeued via label removal", "repo", repo.FullName(), "issue", is.Number)
|
||||
if err := d.Store.IncrementRetryCount(ctx, task.ID); err != nil {
|
||||
d.Log.Error("increment retry_count failed", "issue", is.Number, "err", err)
|
||||
}
|
||||
d.Log.Info("parked task requeued via label removal", "repo", repo.FullName(), "issue", is.Number, "retry_count", task.RetryCount+1, "max_retries", maxRetries)
|
||||
d.Forge.SetStateLabel(ctx, repo.Owner, repo.Name, is.Number, d.Cfg.Dispatch.StateLabel("queued"), d.Cfg.Dispatch.LabelValues())
|
||||
d.emit(events.Event{
|
||||
Kind: "task_retried", TaskID: task.ID,
|
||||
Repo: repo.FullName(), Issue: is.Number, Detail: "archie:parked label removed",
|
||||
Repo: repo.FullName(), Issue: is.Number, Detail: "parked label removed",
|
||||
})
|
||||
}
|
||||
|
||||
// markDead permanently parks a task that has exhausted its max_retries.
|
||||
func (d *Daemon) markDead(ctx context.Context, repo config.Repo, task *store.Task, maxRetries int) {
|
||||
reason := fmt.Sprintf("max retries reached (%d/%d) — task parked permanently", task.RetryCount, maxRetries)
|
||||
if err := d.Store.Transition(ctx, task.ID, store.StatusParked, store.StatusDead, reason); err != nil {
|
||||
d.Log.Error("transition to dead failed", "task", task.ID, "err", err)
|
||||
return
|
||||
}
|
||||
d.Forge.SetStateLabel(ctx, task.Owner, task.Repo, task.IssueNumber,
|
||||
d.Cfg.Dispatch.StateLabel("dead"), d.Cfg.Dispatch.LabelValues())
|
||||
d.Log.Warn("task permanently parked after max retries",
|
||||
"issue", task.IssueNumber, "retry_count", task.RetryCount, "max_retries", maxRetries)
|
||||
body := fmt.Sprintf("**Max retries reached (%d/%d) — task parked permanently.**\n\n"+
|
||||
"The task failed %d times and has been moved to `dead` status. "+
|
||||
"Manual intervention is required to recover this task.",
|
||||
task.RetryCount, maxRetries, task.RetryCount)
|
||||
if _, err := d.Forge.Comment(ctx, task.Owner, task.Repo, task.IssueNumber, body); err != nil {
|
||||
d.Log.Error("failed to post dead comment", "issue", task.IssueNumber, "err", err)
|
||||
}
|
||||
d.emit(events.Event{
|
||||
Kind: events.KindTaskDead, TaskID: task.ID,
|
||||
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,16 @@
|
||||
package daemon
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
func TestHasLabelUsesExactNames(t *testing.T) {
|
||||
labels := []string{"bug", "archie:parked-old", "feature", "custom,label"}
|
||||
@@ -15,3 +25,214 @@ func TestHasLabelUsesExactNames(t *testing.T) {
|
||||
t.Fatal("hasLabel did not match the exact label name")
|
||||
}
|
||||
}
|
||||
|
||||
// testForge implements forge.Forge for daemon tests.
|
||||
type testForge struct {
|
||||
comments []commentCall
|
||||
stateLabels []stateLabelCall
|
||||
}
|
||||
|
||||
type commentCall struct {
|
||||
owner, repo string
|
||||
number int
|
||||
body string
|
||||
}
|
||||
|
||||
type stateLabelCall struct {
|
||||
owner, repo string
|
||||
number int
|
||||
label string
|
||||
knownLabels []string
|
||||
}
|
||||
|
||||
func (f *testForge) Comment(_ context.Context, owner, repo string, number int, body string) (int64, error) {
|
||||
f.comments = append(f.comments, commentCall{owner, repo, number, body})
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (f *testForge) SetStateLabel(_ context.Context, owner, repo string, number int, label string, knownLabels []string) {
|
||||
f.stateLabels = append(f.stateLabels, stateLabelCall{owner, repo, number, label, knownLabels})
|
||||
}
|
||||
|
||||
func (f *testForge) AcceptInvitations(context.Context) error { return nil }
|
||||
func (f *testForge) AssignedIssues(context.Context, string, string, string) ([]forge.Issue, error) {
|
||||
panic("unexpected call")
|
||||
}
|
||||
func (f *testForge) IssuesWithLabel(context.Context, string, string, string) ([]forge.Issue, error) {
|
||||
panic("unexpected call")
|
||||
}
|
||||
func (f *testForge) RepliesAfter(context.Context, string, string, int, int64, string) ([]forge.Reply, error) {
|
||||
panic("unexpected call")
|
||||
}
|
||||
func (f *testForge) CreatePR(context.Context, string, string, string, string, string, string) (int, error) {
|
||||
panic("unexpected call")
|
||||
}
|
||||
func (f *testForge) PRState(context.Context, string, string, int) (string, error) { panic("unexpected call") }
|
||||
func (f *testForge) CloseIssue(context.Context, string, string, int, string) error { panic("unexpected call") }
|
||||
func (f *testForge) React(context.Context, string, string, int, string) error { return nil }
|
||||
func (f *testForge) VerifyPush(context.Context, string, string) error { return nil }
|
||||
|
||||
func testDaemon(t *testing.T, maxRetries int, _ int) (*Daemon, *store.Store, *testForge) {
|
||||
t.Helper()
|
||||
|
||||
s := store.OpenTest(t)
|
||||
fg := &testForge{}
|
||||
|
||||
cfg := config.Config{
|
||||
MaxRetries: maxRetries,
|
||||
Dispatch: config.Dispatch{
|
||||
Labels: map[string]string{
|
||||
"parked": "archie:parked",
|
||||
"queued": "archie:queued",
|
||||
"dead": "archie:dead",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
|
||||
return &Daemon{
|
||||
Cfg: cfg,
|
||||
Store: s,
|
||||
Forge: fg,
|
||||
Log: log,
|
||||
}, s, fg
|
||||
}
|
||||
|
||||
func setupParkedTask(t *testing.T, s *store.Store, retryCount int) *store.Task {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := s.EnqueueIssue(ctx, "sam", "todo", 1, "t", "b", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := s.ClaimNext(ctx)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("claim = (%v, %v)", task, err)
|
||||
}
|
||||
if err := s.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "parked"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range retryCount {
|
||||
if err := s.IncrementRetryCount(ctx, task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
task, err = s.TaskByIssue(ctx, "sam", "todo", 1)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("TaskByIssue = (%+v, %v)", task, err)
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
func TestMaybeRetryParkedUnderThreshold(t *testing.T) {
|
||||
d, s, fg := testDaemon(t, 3, 0)
|
||||
ctx := context.Background()
|
||||
|
||||
_ = setupParkedTask(t, s, 0)
|
||||
|
||||
is := forge.Issue{Number: 1, Labels: []string{}}
|
||||
repo := config.Repo{Owner: "sam", Name: "todo"}
|
||||
|
||||
d.maybeRetryParked(ctx, repo, is)
|
||||
|
||||
task, err := s.TaskByIssue(ctx, "sam", "todo", 1)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("TaskByIssue = (%+v, %v)", task, err)
|
||||
}
|
||||
if task.Status != store.StatusQueued {
|
||||
t.Fatalf("expected status queued, got %s", task.Status)
|
||||
}
|
||||
if task.RetryCount != 1 {
|
||||
t.Fatalf("expected retry_count=1, got %d", task.RetryCount)
|
||||
}
|
||||
if len(fg.stateLabels) != 1 || fg.stateLabels[0].label != "archie:queued" {
|
||||
t.Fatalf("expected queued state label, got %+v", fg.stateLabels)
|
||||
}
|
||||
if len(fg.comments) != 0 {
|
||||
t.Fatalf("expected no comments, got %d", len(fg.comments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRetryParkedAtThreshold(t *testing.T) {
|
||||
d, s, fg := testDaemon(t, 3, 0)
|
||||
ctx := context.Background()
|
||||
|
||||
_ = setupParkedTask(t, s, 3)
|
||||
|
||||
is := forge.Issue{Number: 1, Labels: []string{}}
|
||||
repo := config.Repo{Owner: "sam", Name: "todo"}
|
||||
|
||||
d.maybeRetryParked(ctx, repo, is)
|
||||
|
||||
task, err := s.TaskByIssue(ctx, "sam", "todo", 1)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("TaskByIssue = (%+v, %v)", task, err)
|
||||
}
|
||||
if task.Status != store.StatusDead {
|
||||
t.Fatalf("expected status dead, got %s", task.Status)
|
||||
}
|
||||
if len(fg.stateLabels) != 1 || fg.stateLabels[0].label != "archie:dead" {
|
||||
t.Fatalf("expected dead state label, got %+v", fg.stateLabels)
|
||||
}
|
||||
if len(fg.comments) != 1 || !strings.Contains(fg.comments[0].body, "Max retries reached") {
|
||||
t.Fatalf("expected dead comment, got %+v", fg.comments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRetryParkedSkipsDead(t *testing.T) {
|
||||
d, s, fg := testDaemon(t, 3, 0)
|
||||
ctx := context.Background()
|
||||
|
||||
task := setupParkedTask(t, s, 0)
|
||||
if err := s.Transition(ctx, task.ID, store.StatusParked, store.StatusDead, "manual"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
is := forge.Issue{Number: 1, Labels: []string{}}
|
||||
repo := config.Repo{Owner: "sam", Name: "todo"}
|
||||
|
||||
d.maybeRetryParked(ctx, repo, is)
|
||||
|
||||
if len(fg.stateLabels) != 0 {
|
||||
t.Fatalf("expected no state label calls, got %d", len(fg.stateLabels))
|
||||
}
|
||||
if len(fg.comments) != 0 {
|
||||
t.Fatalf("expected no comments, got %d", len(fg.comments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRetryParkedStillParked(t *testing.T) {
|
||||
d, _, fg := testDaemon(t, 3, 0)
|
||||
ctx := context.Background()
|
||||
|
||||
is := forge.Issue{Number: 1, Labels: []string{"archie:parked", "bug"}}
|
||||
repo := config.Repo{Owner: "sam", Name: "todo"}
|
||||
|
||||
d.maybeRetryParked(ctx, repo, is)
|
||||
|
||||
if len(fg.stateLabels) != 0 {
|
||||
t.Fatalf("expected no state label calls, got %d", len(fg.stateLabels))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaybeRetryParkedRepoOverride(t *testing.T) {
|
||||
d, s, _ := testDaemon(t, 3, 1)
|
||||
ctx := context.Background()
|
||||
|
||||
_ = setupParkedTask(t, s, 1)
|
||||
|
||||
is := forge.Issue{Number: 1, Labels: []string{}}
|
||||
repo := config.Repo{Owner: "sam", Name: "todo", MaxRetries: 1}
|
||||
|
||||
d.maybeRetryParked(ctx, repo, is)
|
||||
|
||||
task, err := s.TaskByIssue(ctx, "sam", "todo", 1)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("TaskByIssue = (%+v, %v)", task, err)
|
||||
}
|
||||
if task.Status != store.StatusDead {
|
||||
t.Fatalf("expected status dead with repo override, got %s", task.Status)
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ const (
|
||||
KindOutcome = "outcome" // data: status, detail
|
||||
KindPRMerged = "pr_merged"
|
||||
KindPRRejected = "pr_rejected"
|
||||
KindTaskDead = "task_dead"
|
||||
KindLog = "log" // data: level, msg
|
||||
)
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ func (s *Store) TaskEvents(ctx context.Context, taskID int64) ([]events.Event, e
|
||||
func (s *Store) Tasks(ctx context.Context, limit int) (tasks []Task, retErr error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, owner, repo, issue_number, title, status, workflow, stage,
|
||||
pr_number, tokens_used, iterations, attempt, park_reason
|
||||
pr_number, tokens_used, iterations, attempt, park_reason, retry_count
|
||||
FROM tasks ORDER BY updated_at DESC LIMIT ?`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -112,7 +112,7 @@ func (s *Store) Tasks(ctx context.Context, limit int) (tasks []Task, retErr erro
|
||||
var t Task
|
||||
if err := rows.Scan(&t.ID, &t.Owner, &t.Repo, &t.IssueNumber, &t.Title,
|
||||
&t.Status, &t.Workflow, &t.Stage, &t.PRNumber, &t.TokensUsed,
|
||||
&t.Iterations, &t.Attempt, &t.ParkReason); err != nil {
|
||||
&t.Iterations, &t.Attempt, &t.ParkReason, &t.RetryCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks = append(tasks, t)
|
||||
|
||||
@@ -88,6 +88,28 @@ func TestWaitingTasksAndRequeue(t *testing.T) {
|
||||
if retried.ParkReason != "" {
|
||||
t.Fatalf("Requeue must clear park_reason, got %q", retried.ParkReason)
|
||||
}
|
||||
|
||||
// Verify retry_count is NOT reset by Requeue (it persists).
|
||||
if err := s.IncrementRetryCount(ctx, task2.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.IncrementRetryCount(ctx, task2.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Park again and requeue.
|
||||
if err := s.Transition(ctx, task2.ID, StatusQueued, StatusParked, "parked again"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.Requeue(ctx, task2.ID, StatusParked, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterRequeue, err := s.TaskByIssue(ctx, "sam", "archie", 1)
|
||||
if err != nil || afterRequeue == nil {
|
||||
t.Fatalf("TaskByIssue after second requeue = (%+v, %v)", afterRequeue, err)
|
||||
}
|
||||
if afterRequeue.RetryCount != 2 {
|
||||
t.Fatalf("Requeue must not reset retry_count, got %d, want 2", afterRequeue.RetryCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTasksListingAndStatusCounts(t *testing.T) {
|
||||
|
||||
+31
-4
@@ -23,6 +23,7 @@ const (
|
||||
StatusPROpen = "pr_open"
|
||||
StatusMerged = "merged"
|
||||
StatusParked = "parked"
|
||||
StatusDead = "dead"
|
||||
StatusRejected = "rejected"
|
||||
StatusClosedWontDo = "closed_wont_do"
|
||||
)
|
||||
@@ -46,6 +47,10 @@ type Task struct {
|
||||
Iterations int
|
||||
Attempt int
|
||||
ParkReason string
|
||||
// RetryCount tracks how many times a parked task has been retried
|
||||
// (parking-to-queued transitions). When it reaches the configured
|
||||
// max_retries the daemon moves the task to StatusDead.
|
||||
RetryCount int
|
||||
// WatchCommentID: replies to the issue after this comment are the
|
||||
// human input a waiting_human task is blocked on.
|
||||
WatchCommentID int64
|
||||
@@ -70,6 +75,10 @@ func Open(path string) (*Store, error) {
|
||||
!strings.Contains(err.Error(), "duplicate column") {
|
||||
return nil, errors.Join(fmt.Errorf("store: migrate: %w", err), db.Close())
|
||||
}
|
||||
if _, err := db.Exec(`ALTER TABLE tasks ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0`); err != nil &&
|
||||
!strings.Contains(err.Error(), "duplicate column") {
|
||||
return nil, errors.Join(fmt.Errorf("store: migrate: %w", err), db.Close())
|
||||
}
|
||||
return &Store{db: db}, nil
|
||||
}
|
||||
|
||||
@@ -109,6 +118,23 @@ CREATE TABLE IF NOT EXISTS transitions (
|
||||
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
// OpenTest opens an in-memory SQLite store suitable for tests.
|
||||
func OpenTest(t interface{ TempDir() string }) *Store {
|
||||
s, err := Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// IncrementRetryCount atomically bumps retry_count by 1. Called after
|
||||
// a parked task is requeued by maybeRetryParked.
|
||||
func (s *Store) IncrementRetryCount(ctx context.Context, taskID int64) error {
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
`UPDATE tasks SET retry_count = retry_count + 1, updated_at = datetime('now') WHERE id = ?`, taskID)
|
||||
return err
|
||||
}
|
||||
|
||||
// EnqueueIssue inserts a new queued task for the issue; returns false if
|
||||
// the issue is already tracked (the idempotency key is owner/repo/number).
|
||||
func (s *Store) EnqueueIssue(ctx context.Context, owner, repo string, number int, title, body, labels string) (bool, error) {
|
||||
@@ -132,7 +158,7 @@ func (s *Store) ClaimNext(ctx context.Context) (*Task, error) {
|
||||
WHERE id = (SELECT id FROM tasks WHERE status='queued' ORDER BY id LIMIT 1)
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status,
|
||||
workflow, stage, branch, plan, notes, pr_number, tokens_used,
|
||||
iterations, attempt, park_reason`)
|
||||
iterations, attempt, park_reason, watch_comment_id, retry_count`)
|
||||
t, err := scanTask(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
@@ -144,7 +170,8 @@ func scanTask(row *sql.Row) (*Task, error) {
|
||||
var t Task
|
||||
err := row.Scan(&t.ID, &t.Owner, &t.Repo, &t.IssueNumber, &t.Title, &t.Body,
|
||||
&t.Labels, &t.Status, &t.Workflow, &t.Stage, &t.Branch, &t.Plan, &t.Notes,
|
||||
&t.PRNumber, &t.TokensUsed, &t.Iterations, &t.Attempt, &t.ParkReason)
|
||||
&t.PRNumber, &t.TokensUsed, &t.Iterations, &t.Attempt, &t.ParkReason,
|
||||
&t.WatchCommentID, &t.RetryCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -161,7 +188,7 @@ func (s *Store) ClaimByIssue(ctx context.Context, owner, repo string, number int
|
||||
WHERE owner=? AND repo=? AND issue_number=? AND status='queued'
|
||||
RETURNING id, owner, repo, issue_number, title, body, labels, status,
|
||||
workflow, stage, branch, plan, notes, pr_number, tokens_used,
|
||||
iterations, attempt, park_reason`, owner, repo, number)
|
||||
iterations, attempt, park_reason, watch_comment_id, retry_count`, owner, repo, number)
|
||||
t, err := scanTask(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
@@ -199,7 +226,7 @@ func (s *Store) TaskByIssue(ctx context.Context, owner, repo string, number int)
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, owner, repo, issue_number, title, body, labels, status,
|
||||
workflow, stage, branch, plan, notes, pr_number, tokens_used,
|
||||
iterations, attempt, park_reason
|
||||
iterations, attempt, park_reason, watch_comment_id, retry_count
|
||||
FROM tasks WHERE owner=? AND repo=? AND issue_number=?`, owner, repo, number)
|
||||
t, err := scanTask(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
|
||||
@@ -71,6 +71,58 @@ func TestEventLogRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncrementRetryCount(t *testing.T) {
|
||||
s := openTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := s.EnqueueIssue(ctx, "sam", "todo", 1, "t", "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task, err := s.ClaimNext(ctx)
|
||||
if err != nil || task == nil {
|
||||
t.Fatalf("claim = (%v, %v)", task, err)
|
||||
}
|
||||
|
||||
if task.RetryCount != 0 {
|
||||
t.Fatalf("expected retry_count=0, got %d", task.RetryCount)
|
||||
}
|
||||
|
||||
// Increment twice and verify.
|
||||
if err := s.IncrementRetryCount(ctx, task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.IncrementRetryCount(ctx, task.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := s.TaskByIssue(ctx, "sam", "todo", 1)
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("TaskByIssue = (%+v, %v)", got, err)
|
||||
}
|
||||
if got.RetryCount != 2 {
|
||||
t.Fatalf("expected retry_count=2, got %d", got.RetryCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryCountColumn(t *testing.T) {
|
||||
s := openTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
if _, err := s.EnqueueIssue(ctx, "sam", "todo", 1, "t", "", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify column exists and defaults to 0 via direct query.
|
||||
var count int
|
||||
if err := s.db.QueryRowContext(ctx,
|
||||
`SELECT retry_count FROM tasks WHERE owner='sam' AND repo='todo' AND issue_number=1`).Scan(&count); err != nil {
|
||||
t.Fatalf("retry_count scan failed: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("default retry_count = %d, want 0", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimTransitionAndRecovery(t *testing.T) {
|
||||
s := openTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -217,6 +217,8 @@ func stateLabelFor(d config.Dispatch, status string) string {
|
||||
return d.StateLabel("waiting")
|
||||
case store.StatusParked:
|
||||
return d.StateLabel("parked")
|
||||
case store.StatusDead:
|
||||
return d.StateLabel("dead")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user