From 52b5740c39d7868f56b2f0fc933294f64aac0a7a Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sun, 8 Nov 2026 17:23:30 +1100 Subject: [PATCH] fix(daemon): cap tasks per issue poll and wait for disk before cloning (#1805) --- internal/config/config.go | 4 ++ internal/daemon/daemon.go | 49 ++++++++++--- internal/daemon/intake_cap_test.go | 71 +++++++++++++++++++ .../infrastructure/configuration/defaults.go | 4 ++ .../controlplanestore/controlplane.go | 2 +- .../controlplanestore/operational_settings.go | 11 ++- .../controlplanestore/runtime_config.go | 3 + internal/worktree/space.go | 54 ++++++++++++++ internal/worktree/space_test.go | 34 +++++++++ ui/src/settings/SchedulingPolicyPage.vue | 20 ++++++ 10 files changed, 239 insertions(+), 13 deletions(-) create mode 100644 internal/daemon/intake_cap_test.go create mode 100644 internal/worktree/space.go create mode 100644 internal/worktree/space_test.go diff --git a/internal/config/config.go b/internal/config/config.go index f2d4763b4..009eddf51 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -370,6 +370,10 @@ type Config struct { // MaxRetries caps how many times a parked task is retried before // being permanently parked (status "dead"). Defaults to 3. MaxRetries int `toml:"max_retries" yaml:"max_retries"` + // IntakePerPass caps how many tasks one issue poll creates, so a backlog + // found at once (the first poll after an outage) drains over several + // passes. Defaults to 20. + IntakePerPass int `toml:"intake_per_pass" yaml:"intake_per_pass"` // StateDir is the host directory archie keeps non-database state in: the // embedded NATS store and its endpoint file, the task-log registry, and // the readiness disk probe's data target. Bootstrap-only. diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 1fec3ed63..8e70b48f3 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -304,6 +304,14 @@ func (d *Daemon) Startup(ctx context.Context) error { // an unreachable host would otherwise hold boot open indefinitely. const sweepTimeout = 2 * time.Minute +// A clone waits, holding its slot, until the work volume has minCloneSpace +// free, and parks as transient if it does not within diskWaitLimit. +const ( + minCloneSpace = 5 << 30 + diskCheckEvery = 15 * time.Second + diskWaitLimit = 10 * time.Minute +) + // sweepAccess accepts pending invitations and verifies push access through // every organisation connection, each in its own goroutine. func (d *Daemon) sweepAccess(ctx context.Context) { @@ -1292,6 +1300,7 @@ func (d *Daemon) poll(ctx context.Context) error { return nil } var failed []string + room := d.Cfg.Get().IntakePerPass for _, source := range d.issueSources(ctx) { if !d.identityActive(ctx, source.Identity) { continue @@ -1309,10 +1318,8 @@ func (d *Daemon) poll(ctx context.Context) error { } else { failed = append(failed, repo.FullName()) } - for _, is := range issues { - if d.Tasks != nil { - d.pollNATS(ctx, source.Forge, cfg, repo, is, strings.Join(is.Labels, ","), tag) - } + if d.Tasks != nil { + d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room) } } } @@ -1389,16 +1396,34 @@ func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identit } } -// pollNATS publishes discovered issues to NATS (new flow). -func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) { +// publishDiscovered publishes a repo's discovered issues until the pass has +// created its cap of tasks. room is what is left of the cap; the issues left +// over are found again by the next poll, because an issue with a task is +// skipped and one without is not. A cap of zero or less is no cap. +func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) { + capped := *room > 0 + for _, is := range issues { + if capped && *room == 0 { + d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName()) + return + } + if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped { + *room-- + } + } +} + +// pollNATS publishes a discovered issue to NATS and reports whether it created +// a task. +func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool { // Read-only existence check prevents rediscovering an existing task. existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number) if err != nil { d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err) - return + return false } if existing != nil { - return + return false } // Classify and encode here: the envelope, its routing kind and its // subject belong to the work-intake domain, not to the transport. @@ -1415,9 +1440,10 @@ func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config } if err := d.PublishTask(ctx, task); err != nil { d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err) - return + return false } d.acknowledge(ctx, fg, cfg, repo, is) + return true } // PublishTask validates a task envelope, stamps the producing identity's @@ -1795,6 +1821,11 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree } target = worktree.Target(task.Branch) } + if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil { + d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err) + d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient) + return "", false + } // Every task gets an independent full clone. prepare := func() (string, string, error) { return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target) diff --git a/internal/daemon/intake_cap_test.go b/internal/daemon/intake_cap_test.go new file mode 100644 index 000000000..ddad7d99f --- /dev/null +++ b/internal/daemon/intake_cap_test.go @@ -0,0 +1,71 @@ +package daemon + +import ( + "context" + "log/slog" + "testing" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/forge" +) + +type taskLookup struct { + storecontract.TaskStore + have map[int]bool +} + +func (s taskLookup) TaskByIssue(_ context.Context, _, _ string, number int) (*workflow.Task, error) { + if s.have[number] { + return &workflow.Task{IssueNumber: number}, nil + } + return nil, nil +} + +type publishCounter struct { + stubTasks + published int +} + +func (p *publishCounter) PublishUnique(context.Context, string, string, []byte) error { + p.published++ + return nil +} + +// One poll creates at most the cap of tasks; the rest wait for a later poll, +// and an issue that already has a task neither counts nor is created twice. +func TestPublishDiscoveredCap(t *testing.T) { + tests := []struct { + name string + issues int + existing []int + cap int + want int + }{ + {"cap below backlog", 10, nil, 3, 3}, + {"backlog below cap", 2, nil, 3, 2}, + {"existing tasks do not use the cap", 6, []int{1, 2}, 3, 3}, + {"no cap", 10, nil, 0, 10}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bus := &publishCounter{} + have := map[int]bool{} + for _, n := range tt.existing { + have[n] = true + } + d := &Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus} + var issues []forge.Issue + for n := 1; n <= tt.issues; n++ { + issues = append(issues, forge.Issue{Number: n, Labels: []string{"archie"}}) + } + room := tt.cap + repo := config.Repo{Owner: "acme", Name: "widget"} + d.publishDiscovered(t.Context(), nil, config.Config{}, repo, issues, "", &room) + if bus.published != tt.want { + t.Fatalf("published %d tasks, want %d", bus.published, tt.want) + } + }) + } +} diff --git a/internal/infrastructure/configuration/defaults.go b/internal/infrastructure/configuration/defaults.go index 9e76ed84c..26a482a76 100644 --- a/internal/infrastructure/configuration/defaults.go +++ b/internal/infrastructure/configuration/defaults.go @@ -15,6 +15,7 @@ const ( defaultPollInterval = 60 * time.Second defaultWebListen = "127.0.0.1:8484" // "off" disables defaultMaxRetries = 3 + defaultIntakePerPass = 20 defaultForgeType = "github" defaultForgeHost = "https://github.com" defaultDispatchTrigger = "assignee" @@ -225,6 +226,9 @@ func (l *Loader) applyGeneralDefaults(cfg *config.Config) { if cfg.MaxRetries == 0 { cfg.MaxRetries = defaultMaxRetries } + if cfg.IntakePerPass == 0 { + cfg.IntakePerPass = defaultIntakePerPass + } } func expandHomePath(path string) string { diff --git a/internal/infrastructure/controlplanestore/controlplane.go b/internal/infrastructure/controlplanestore/controlplane.go index 32fe1e687..1e199de25 100644 --- a/internal/infrastructure/controlplanestore/controlplane.go +++ b/internal/infrastructure/controlplanestore/controlplane.go @@ -204,7 +204,7 @@ var orgSeededKinds = map[string]bool{ var orgEmptyDocuments = map[string]string{ ChannelSettingsKind: `{"operator":"","show_tool_calls":false,"max_steps":0,"allowed_user_ids":[],"rate_limit":{"window":0,"max_requests":0},"unrestricted_filesystem":false,"workspace":"","agent":""}`, TaskPolicyKind: `{"notify_webhook":"","notify_min_severity":"","notify_secret":{}}`, - SchedulingPolicyKind: `{"poll_interval":"","max_retries":0,"dispatch":{"trigger":"","ack_reaction":""}}`, + SchedulingPolicyKind: `{"poll_interval":"","max_retries":0,"intake_per_pass":0,"dispatch":{"trigger":"","ack_reaction":""}}`, } // resource seeds an org's workflow documents on first use, including orgs diff --git a/internal/infrastructure/controlplanestore/operational_settings.go b/internal/infrastructure/controlplanestore/operational_settings.go index 03ad268ac..570b382ae 100644 --- a/internal/infrastructure/controlplanestore/operational_settings.go +++ b/internal/infrastructure/controlplanestore/operational_settings.go @@ -27,6 +27,8 @@ const ( type schedulingPolicy struct { PollInterval string `json:"poll_interval"` MaxRetries int `json:"max_retries"` + // IntakePerPass caps the tasks one issue poll creates. + IntakePerPass int `json:"intake_per_pass"` // Label is the dispatch trigger label. Nil keeps the file's label. Label *string `json:"label,omitempty"` Dispatch config.Dispatch `json:"dispatch"` @@ -46,7 +48,7 @@ func operationalDefinitions() []Definition { // everywhere the record reports it. {Kind: ChannelSettingsKind, Title: "Channel settings", ApplyMode: "restart-required", Document: channelSettings{}, Seed: seedChannels, Validate: validateChannels, Normalize: normalizeChannels}, {Kind: SchedulingPolicyKind, Title: "Scheduling policy", ApplyMode: "live", Document: schedulingPolicy{}, Seed: func(cfg config.Config) any { - return schedulingPolicy{PollInterval: cfg.PollInterval.Std().String(), MaxRetries: cfg.MaxRetries, Label: &cfg.Label, Dispatch: cfg.Dispatch} + return schedulingPolicy{PollInterval: cfg.PollInterval.Std().String(), MaxRetries: cfg.MaxRetries, IntakePerPass: cfg.IntakePerPass, Label: &cfg.Label, Dispatch: cfg.Dispatch} }, Validate: validateScheduling}, // Live: changed MCP servers are reconnected, rolling back to the old engine // if the new one fails to start. @@ -117,6 +119,9 @@ func validateScheduling(input []byte) error { if policy.MaxRetries < 0 { return fmt.Errorf("max_retries must not be negative") } + if policy.IntakePerPass < 0 { + return fmt.Errorf("intake_per_pass must not be negative") + } if policy.PollInterval != "" { interval, err := time.ParseDuration(policy.PollInterval) if err != nil || interval <= 0 { @@ -150,8 +155,8 @@ func checkSchedulingWrite(orgID string, value []byte) error { } return nil } - if policy.PollInterval != "" || policy.MaxRetries != 0 { - return fmt.Errorf("%w: poll_interval and max_retries are instance settings; an org sets the dispatch rule and label only", ErrValidation) + if policy.PollInterval != "" || policy.MaxRetries != 0 || policy.IntakePerPass != 0 { + return fmt.Errorf("%w: poll_interval, max_retries and intake_per_pass are instance settings; an org sets the dispatch rule and label only", ErrValidation) } return nil } diff --git a/internal/infrastructure/controlplanestore/runtime_config.go b/internal/infrastructure/controlplanestore/runtime_config.go index 696aa0858..c9fac7bb9 100644 --- a/internal/infrastructure/controlplanestore/runtime_config.go +++ b/internal/infrastructure/controlplanestore/runtime_config.go @@ -132,6 +132,9 @@ func layerScheduling(out *config.Config, policy schedulingPolicy) error { if policy.MaxRetries != 0 { out.MaxRetries = policy.MaxRetries } + if policy.IntakePerPass != 0 { + out.IntakePerPass = policy.IntakePerPass + } if policy.Dispatch.Trigger != "" { out.Dispatch.Trigger = policy.Dispatch.Trigger } diff --git a/internal/worktree/space.go b/internal/worktree/space.go new file mode 100644 index 000000000..692c29ab2 --- /dev/null +++ b/internal/worktree/space.go @@ -0,0 +1,54 @@ +package worktree + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "syscall" + "time" +) + +// ErrDiskLow means the work volume stayed below the free-space floor. +var ErrDiskLow = errors.New("disk low") + +// FreeBytes reports the space available to unprivileged writers on the volume +// holding the work directory, which may not exist yet. +func (m *Manager) FreeBytes() (uint64, error) { + path := m.WorkDir + for { + var st syscall.Statfs_t + err := syscall.Statfs(path, &st) + if err == nil { + return st.Bavail * uint64(st.Bsize), nil //nolint:gosec // Bsize is positive + } + parent := filepath.Dir(path) + if !errors.Is(err, os.ErrNotExist) || parent == path { + return 0, err + } + path = parent + } +} + +// AwaitSpace blocks until free reports at least floor bytes, checking every +// interval for up to wait, so a full volume delays a clone instead of failing +// it half-written. A volume that cannot be measured does not block the clone. +func AwaitSpace(ctx context.Context, free func() (uint64, error), floor uint64, interval, wait time.Duration) error { + deadline := time.Now().Add(wait) + for { + have, err := free() + if err == nil && have < floor { + if !time.Now().Add(interval).Before(deadline) { + return fmt.Errorf("%w: %d MiB free on the work volume, %d MiB needed to clone", ErrDiskLow, have>>20, floor>>20) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + continue + } + return nil + } +} diff --git a/internal/worktree/space_test.go b/internal/worktree/space_test.go new file mode 100644 index 000000000..0f5d28d73 --- /dev/null +++ b/internal/worktree/space_test.go @@ -0,0 +1,34 @@ +package worktree + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestAwaitSpace(t *testing.T) { + tests := []struct { + name string + reports []uint64 + wantErr error + }{ + {"enough at once", []uint64{10}, nil}, + {"low then recovers", []uint64{1, 1, 10}, nil}, + {"stays low", []uint64{1}, ErrDiskLow}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + calls := 0 + free := func() (uint64, error) { + i := min(calls, len(tt.reports)-1) + calls++ + return tt.reports[i], nil + } + err := AwaitSpace(context.Background(), free, 5, time.Millisecond, 20*time.Millisecond) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + }) + } +} diff --git a/ui/src/settings/SchedulingPolicyPage.vue b/ui/src/settings/SchedulingPolicyPage.vue index dd15e8b71..260925192 100644 --- a/ui/src/settings/SchedulingPolicyPage.vue +++ b/ui/src/settings/SchedulingPolicyPage.vue @@ -28,6 +28,7 @@ const KIND = "scheduling-policy"; interface SchedulingPolicy { poll_interval: string; max_retries: number; + intake_per_pass: number; label?: string; dispatch: { trigger: string; ack_reaction: string }; } @@ -147,6 +148,25 @@ const triggers = [ + + + + Tasks per poll + Most tasks one poll creates. The rest follow on later polls. + +
+ + + + + + + + +
+
+
+ Poll interval