fix(daemon): cap tasks per issue poll and wait for disk before cloning (#1805)
deploy / plan (push) Successful in 1m6s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s
deploy / gate (push) Canceled after 11s

This commit is contained in:
sam committed 2026-11-08 17:26:00 +11:00
1 parent d11a0cd245
commit 52b5740c39
10 files changed
+239 -13

No files matched your search

+4
View File
@@ -370,6 +370,10 @@ type Config struct {
// MaxRetries caps how many times a parked task is retried before
// being permanently parked (status "dead"). Defaults to 3.
MaxRetries int `toml:"max_retries" yaml:"max_retries"`
// IntakePerPass caps how many tasks one issue poll creates, so a backlog
// found at once (the first poll after an outage) drains over several
// passes. Defaults to 20.
IntakePerPass int `toml:"intake_per_pass" yaml:"intake_per_pass"`
// StateDir is the host directory archie keeps non-database state in: the
// embedded NATS store and its endpoint file, the task-log registry, and
// the readiness disk probe's data target. Bootstrap-only.
+40 -9
View File
@@ -304,6 +304,14 @@ func (d *Daemon) Startup(ctx context.Context) error {
// an unreachable host would otherwise hold boot open indefinitely.
const sweepTimeout = 2 * time.Minute
// A clone waits, holding its slot, until the work volume has minCloneSpace
// free, and parks as transient if it does not within diskWaitLimit.
const (
minCloneSpace = 5 << 30
diskCheckEvery = 15 * time.Second
diskWaitLimit = 10 * time.Minute
)
// sweepAccess accepts pending invitations and verifies push access through
// every organisation connection, each in its own goroutine.
func (d *Daemon) sweepAccess(ctx context.Context) {
@@ -1292,6 +1300,7 @@ func (d *Daemon) poll(ctx context.Context) error {
return nil
}
var failed []string
room := d.Cfg.Get().IntakePerPass
for _, source := range d.issueSources(ctx) {
if !d.identityActive(ctx, source.Identity) {
continue
@@ -1309,10 +1318,8 @@ func (d *Daemon) poll(ctx context.Context) error {
} else {
failed = append(failed, repo.FullName())
}
for _, is := range issues {
if d.Tasks != nil {
d.pollNATS(ctx, source.Forge, cfg, repo, is, strings.Join(is.Labels, ","), tag)
}
if d.Tasks != nil {
d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room)
}
}
}
@@ -1389,16 +1396,34 @@ func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identit
}
}
// pollNATS publishes discovered issues to NATS (new flow).
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) {
// publishDiscovered publishes a repo's discovered issues until the pass has
// created its cap of tasks. room is what is left of the cap; the issues left
// over are found again by the next poll, because an issue with a task is
// skipped and one without is not. A cap of zero or less is no cap.
func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) {
capped := *room > 0
for _, is := range issues {
if capped && *room == 0 {
d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName())
return
}
if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped {
*room--
}
}
}
// pollNATS publishes a discovered issue to NATS and reports whether it created
// a task.
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool {
// Read-only existence check prevents rediscovering an existing task.
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
if err != nil {
d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return
return false
}
if existing != nil {
return
return false
}
// Classify and encode here: the envelope, its routing kind and its
// subject belong to the work-intake domain, not to the transport.
@@ -1415,9 +1440,10 @@ func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config
}
if err := d.PublishTask(ctx, task); err != nil {
d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return
return false
}
d.acknowledge(ctx, fg, cfg, repo, is)
return true
}
// PublishTask validates a task envelope, stamps the producing identity's
@@ -1795,6 +1821,11 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
}
target = worktree.Target(task.Branch)
}
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
return "", false
}
// Every task gets an independent full clone.
prepare := func() (string, string, error) {
return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
+71
View File
@@ -0,0 +1,71 @@
package daemon
import (
"context"
"log/slog"
"testing"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/forge"
)
type taskLookup struct {
storecontract.TaskStore
have map[int]bool
}
func (s taskLookup) TaskByIssue(_ context.Context, _, _ string, number int) (*workflow.Task, error) {
if s.have[number] {
return &workflow.Task{IssueNumber: number}, nil
}
return nil, nil
}
type publishCounter struct {
stubTasks
published int
}
func (p *publishCounter) PublishUnique(context.Context, string, string, []byte) error {
p.published++
return nil
}
// One poll creates at most the cap of tasks; the rest wait for a later poll,
// and an issue that already has a task neither counts nor is created twice.
func TestPublishDiscoveredCap(t *testing.T) {
tests := []struct {
name string
issues int
existing []int
cap int
want int
}{
{"cap below backlog", 10, nil, 3, 3},
{"backlog below cap", 2, nil, 3, 2},
{"existing tasks do not use the cap", 6, []int{1, 2}, 3, 3},
{"no cap", 10, nil, 0, 10},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
bus := &publishCounter{}
have := map[int]bool{}
for _, n := range tt.existing {
have[n] = true
}
d := &Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus}
var issues []forge.Issue
for n := 1; n <= tt.issues; n++ {
issues = append(issues, forge.Issue{Number: n, Labels: []string{"archie"}})
}
room := tt.cap
repo := config.Repo{Owner: "acme", Name: "widget"}
d.publishDiscovered(t.Context(), nil, config.Config{}, repo, issues, "", &room)
if bus.published != tt.want {
t.Fatalf("published %d tasks, want %d", bus.published, tt.want)
}
})
}
}
@@ -15,6 +15,7 @@ const (
defaultPollInterval = 60 * time.Second
defaultWebListen = "127.0.0.1:8484" // "off" disables
defaultMaxRetries = 3
defaultIntakePerPass = 20
defaultForgeType = "github"
defaultForgeHost = "https://github.com"
defaultDispatchTrigger = "assignee"
@@ -225,6 +226,9 @@ func (l *Loader) applyGeneralDefaults(cfg *config.Config) {
if cfg.MaxRetries == 0 {
cfg.MaxRetries = defaultMaxRetries
}
if cfg.IntakePerPass == 0 {
cfg.IntakePerPass = defaultIntakePerPass
}
}
func expandHomePath(path string) string {
@@ -204,7 +204,7 @@ var orgSeededKinds = map[string]bool{
var orgEmptyDocuments = map[string]string{
ChannelSettingsKind: `{"operator":"","show_tool_calls":false,"max_steps":0,"allowed_user_ids":[],"rate_limit":{"window":0,"max_requests":0},"unrestricted_filesystem":false,"workspace":"","agent":""}`,
TaskPolicyKind: `{"notify_webhook":"","notify_min_severity":"","notify_secret":{}}`,
SchedulingPolicyKind: `{"poll_interval":"","max_retries":0,"dispatch":{"trigger":"","ack_reaction":""}}`,
SchedulingPolicyKind: `{"poll_interval":"","max_retries":0,"intake_per_pass":0,"dispatch":{"trigger":"","ack_reaction":""}}`,
}
// resource seeds an org's workflow documents on first use, including orgs
@@ -27,6 +27,8 @@ const (
type schedulingPolicy struct {
PollInterval string `json:"poll_interval"`
MaxRetries int `json:"max_retries"`
// IntakePerPass caps the tasks one issue poll creates.
IntakePerPass int `json:"intake_per_pass"`
// Label is the dispatch trigger label. Nil keeps the file's label.
Label *string `json:"label,omitempty"`
Dispatch config.Dispatch `json:"dispatch"`
@@ -46,7 +48,7 @@ func operationalDefinitions() []Definition {
// everywhere the record reports it.
{Kind: ChannelSettingsKind, Title: "Channel settings", ApplyMode: "restart-required", Document: channelSettings{}, Seed: seedChannels, Validate: validateChannels, Normalize: normalizeChannels},
{Kind: SchedulingPolicyKind, Title: "Scheduling policy", ApplyMode: "live", Document: schedulingPolicy{}, Seed: func(cfg config.Config) any {
return schedulingPolicy{PollInterval: cfg.PollInterval.Std().String(), MaxRetries: cfg.MaxRetries, Label: &cfg.Label, Dispatch: cfg.Dispatch}
return schedulingPolicy{PollInterval: cfg.PollInterval.Std().String(), MaxRetries: cfg.MaxRetries, IntakePerPass: cfg.IntakePerPass, Label: &cfg.Label, Dispatch: cfg.Dispatch}
}, Validate: validateScheduling},
// Live: changed MCP servers are reconnected, rolling back to the old engine
// if the new one fails to start.
@@ -117,6 +119,9 @@ func validateScheduling(input []byte) error {
if policy.MaxRetries < 0 {
return fmt.Errorf("max_retries must not be negative")
}
if policy.IntakePerPass < 0 {
return fmt.Errorf("intake_per_pass must not be negative")
}
if policy.PollInterval != "" {
interval, err := time.ParseDuration(policy.PollInterval)
if err != nil || interval <= 0 {
@@ -150,8 +155,8 @@ func checkSchedulingWrite(orgID string, value []byte) error {
}
return nil
}
if policy.PollInterval != "" || policy.MaxRetries != 0 {
return fmt.Errorf("%w: poll_interval and max_retries are instance settings; an org sets the dispatch rule and label only", ErrValidation)
if policy.PollInterval != "" || policy.MaxRetries != 0 || policy.IntakePerPass != 0 {
return fmt.Errorf("%w: poll_interval, max_retries and intake_per_pass are instance settings; an org sets the dispatch rule and label only", ErrValidation)
}
return nil
}
@@ -132,6 +132,9 @@ func layerScheduling(out *config.Config, policy schedulingPolicy) error {
if policy.MaxRetries != 0 {
out.MaxRetries = policy.MaxRetries
}
if policy.IntakePerPass != 0 {
out.IntakePerPass = policy.IntakePerPass
}
if policy.Dispatch.Trigger != "" {
out.Dispatch.Trigger = policy.Dispatch.Trigger
}
+54
View File
@@ -0,0 +1,54 @@
package worktree
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"syscall"
"time"
)
// ErrDiskLow means the work volume stayed below the free-space floor.
var ErrDiskLow = errors.New("disk low")
// FreeBytes reports the space available to unprivileged writers on the volume
// holding the work directory, which may not exist yet.
func (m *Manager) FreeBytes() (uint64, error) {
path := m.WorkDir
for {
var st syscall.Statfs_t
err := syscall.Statfs(path, &st)
if err == nil {
return st.Bavail * uint64(st.Bsize), nil //nolint:gosec // Bsize is positive
}
parent := filepath.Dir(path)
if !errors.Is(err, os.ErrNotExist) || parent == path {
return 0, err
}
path = parent
}
}
// AwaitSpace blocks until free reports at least floor bytes, checking every
// interval for up to wait, so a full volume delays a clone instead of failing
// it half-written. A volume that cannot be measured does not block the clone.
func AwaitSpace(ctx context.Context, free func() (uint64, error), floor uint64, interval, wait time.Duration) error {
deadline := time.Now().Add(wait)
for {
have, err := free()
if err == nil && have < floor {
if !time.Now().Add(interval).Before(deadline) {
return fmt.Errorf("%w: %d MiB free on the work volume, %d MiB needed to clone", ErrDiskLow, have>>20, floor>>20)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(interval):
}
continue
}
return nil
}
}
+34
View File
@@ -0,0 +1,34 @@
package worktree
import (
"context"
"errors"
"testing"
"time"
)
func TestAwaitSpace(t *testing.T) {
tests := []struct {
name string
reports []uint64
wantErr error
}{
{"enough at once", []uint64{10}, nil},
{"low then recovers", []uint64{1, 1, 10}, nil},
{"stays low", []uint64{1}, ErrDiskLow},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
calls := 0
free := func() (uint64, error) {
i := min(calls, len(tt.reports)-1)
calls++
return tt.reports[i], nil
}
err := AwaitSpace(context.Background(), free, 5, time.Millisecond, 20*time.Millisecond)
if !errors.Is(err, tt.wantErr) {
t.Fatalf("err = %v, want %v", err, tt.wantErr)
}
})
}
}
+20
View File
@@ -28,6 +28,7 @@ const KIND = "scheduling-policy";
interface SchedulingPolicy {
poll_interval: string;
max_retries: number;
intake_per_pass: number;
label?: string;
dispatch: { trigger: string; ack_reaction: string };
}
@@ -147,6 +148,25 @@ const triggers = [
</Field>
</form.Field>
<form.Field v-slot="{ field }" name="intake_per_pass">
<Field orientation="responsive">
<FieldContent>
<FieldLabel>Tasks per poll</FieldLabel>
<FieldDescription>Most tasks one poll creates. The rest follow on later polls.</FieldDescription>
</FieldContent>
<div class="flex min-w-0 flex-wrap items-center gap-3 @md/field-group:flex-1">
<NumberField :model-value="field.state.value" :min="0" class="w-32" @update:model-value="field.handleChange">
<NumberFieldContent>
<NumberFieldDecrement />
<NumberFieldInput class="font-mono" aria-label="Tasks per poll" />
<NumberFieldIncrement />
</NumberFieldContent>
</NumberField>
<DraftHint :kind="KIND" path="intake_per_pass" />
</div>
</Field>
</form.Field>
<form.Field v-slot="{ field }" name="poll_interval">
<Field orientation="responsive" :data-invalid="fieldInvalid(field)">
<FieldContent><FieldLabel for="sp-poll">Poll interval</FieldLabel></FieldContent>