fix(daemon): cap tasks per issue poll and wait for disk before cloning (#1805)
This commit is contained in:
10 files changed
+239
-13
No files matched your search
@@ -370,6 +370,10 @@ type Config struct {
|
||||
// MaxRetries caps how many times a parked task is retried before
|
||||
// being permanently parked (status "dead"). Defaults to 3.
|
||||
MaxRetries int `toml:"max_retries" yaml:"max_retries"`
|
||||
// IntakePerPass caps how many tasks one issue poll creates, so a backlog
|
||||
// found at once (the first poll after an outage) drains over several
|
||||
// passes. Defaults to 20.
|
||||
IntakePerPass int `toml:"intake_per_pass" yaml:"intake_per_pass"`
|
||||
// StateDir is the host directory archie keeps non-database state in: the
|
||||
// embedded NATS store and its endpoint file, the task-log registry, and
|
||||
// the readiness disk probe's data target. Bootstrap-only.
|
||||
|
||||
@@ -304,6 +304,14 @@ func (d *Daemon) Startup(ctx context.Context) error {
|
||||
// an unreachable host would otherwise hold boot open indefinitely.
|
||||
const sweepTimeout = 2 * time.Minute
|
||||
|
||||
// A clone waits, holding its slot, until the work volume has minCloneSpace
|
||||
// free, and parks as transient if it does not within diskWaitLimit.
|
||||
const (
|
||||
minCloneSpace = 5 << 30
|
||||
diskCheckEvery = 15 * time.Second
|
||||
diskWaitLimit = 10 * time.Minute
|
||||
)
|
||||
|
||||
// sweepAccess accepts pending invitations and verifies push access through
|
||||
// every organisation connection, each in its own goroutine.
|
||||
func (d *Daemon) sweepAccess(ctx context.Context) {
|
||||
@@ -1292,6 +1300,7 @@ func (d *Daemon) poll(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
var failed []string
|
||||
room := d.Cfg.Get().IntakePerPass
|
||||
for _, source := range d.issueSources(ctx) {
|
||||
if !d.identityActive(ctx, source.Identity) {
|
||||
continue
|
||||
@@ -1309,10 +1318,8 @@ func (d *Daemon) poll(ctx context.Context) error {
|
||||
} else {
|
||||
failed = append(failed, repo.FullName())
|
||||
}
|
||||
for _, is := range issues {
|
||||
if d.Tasks != nil {
|
||||
d.pollNATS(ctx, source.Forge, cfg, repo, is, strings.Join(is.Labels, ","), tag)
|
||||
}
|
||||
if d.Tasks != nil {
|
||||
d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1389,16 +1396,34 @@ func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identit
|
||||
}
|
||||
}
|
||||
|
||||
// pollNATS publishes discovered issues to NATS (new flow).
|
||||
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) {
|
||||
// publishDiscovered publishes a repo's discovered issues until the pass has
|
||||
// created its cap of tasks. room is what is left of the cap; the issues left
|
||||
// over are found again by the next poll, because an issue with a task is
|
||||
// skipped and one without is not. A cap of zero or less is no cap.
|
||||
func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) {
|
||||
capped := *room > 0
|
||||
for _, is := range issues {
|
||||
if capped && *room == 0 {
|
||||
d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName())
|
||||
return
|
||||
}
|
||||
if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped {
|
||||
*room--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pollNATS publishes a discovered issue to NATS and reports whether it created
|
||||
// a task.
|
||||
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool {
|
||||
// Read-only existence check prevents rediscovering an existing task.
|
||||
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
|
||||
if err != nil {
|
||||
d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
if existing != nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
// Classify and encode here: the envelope, its routing kind and its
|
||||
// subject belong to the work-intake domain, not to the transport.
|
||||
@@ -1415,9 +1440,10 @@ func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config
|
||||
}
|
||||
if err := d.PublishTask(ctx, task); err != nil {
|
||||
d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
d.acknowledge(ctx, fg, cfg, repo, is)
|
||||
return true
|
||||
}
|
||||
|
||||
// PublishTask validates a task envelope, stamps the producing identity's
|
||||
@@ -1795,6 +1821,11 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
|
||||
}
|
||||
target = worktree.Target(task.Branch)
|
||||
}
|
||||
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
|
||||
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
|
||||
return "", false
|
||||
}
|
||||
// Every task gets an independent full clone.
|
||||
prepare := func() (string, string, error) {
|
||||
return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/forge"
|
||||
)
|
||||
|
||||
type taskLookup struct {
|
||||
storecontract.TaskStore
|
||||
have map[int]bool
|
||||
}
|
||||
|
||||
func (s taskLookup) TaskByIssue(_ context.Context, _, _ string, number int) (*workflow.Task, error) {
|
||||
if s.have[number] {
|
||||
return &workflow.Task{IssueNumber: number}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type publishCounter struct {
|
||||
stubTasks
|
||||
published int
|
||||
}
|
||||
|
||||
func (p *publishCounter) PublishUnique(context.Context, string, string, []byte) error {
|
||||
p.published++
|
||||
return nil
|
||||
}
|
||||
|
||||
// One poll creates at most the cap of tasks; the rest wait for a later poll,
|
||||
// and an issue that already has a task neither counts nor is created twice.
|
||||
func TestPublishDiscoveredCap(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
issues int
|
||||
existing []int
|
||||
cap int
|
||||
want int
|
||||
}{
|
||||
{"cap below backlog", 10, nil, 3, 3},
|
||||
{"backlog below cap", 2, nil, 3, 2},
|
||||
{"existing tasks do not use the cap", 6, []int{1, 2}, 3, 3},
|
||||
{"no cap", 10, nil, 0, 10},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
bus := &publishCounter{}
|
||||
have := map[int]bool{}
|
||||
for _, n := range tt.existing {
|
||||
have[n] = true
|
||||
}
|
||||
d := &Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus}
|
||||
var issues []forge.Issue
|
||||
for n := 1; n <= tt.issues; n++ {
|
||||
issues = append(issues, forge.Issue{Number: n, Labels: []string{"archie"}})
|
||||
}
|
||||
room := tt.cap
|
||||
repo := config.Repo{Owner: "acme", Name: "widget"}
|
||||
d.publishDiscovered(t.Context(), nil, config.Config{}, repo, issues, "", &room)
|
||||
if bus.published != tt.want {
|
||||
t.Fatalf("published %d tasks, want %d", bus.published, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
defaultPollInterval = 60 * time.Second
|
||||
defaultWebListen = "127.0.0.1:8484" // "off" disables
|
||||
defaultMaxRetries = 3
|
||||
defaultIntakePerPass = 20
|
||||
defaultForgeType = "github"
|
||||
defaultForgeHost = "https://github.com"
|
||||
defaultDispatchTrigger = "assignee"
|
||||
@@ -225,6 +226,9 @@ func (l *Loader) applyGeneralDefaults(cfg *config.Config) {
|
||||
if cfg.MaxRetries == 0 {
|
||||
cfg.MaxRetries = defaultMaxRetries
|
||||
}
|
||||
if cfg.IntakePerPass == 0 {
|
||||
cfg.IntakePerPass = defaultIntakePerPass
|
||||
}
|
||||
}
|
||||
|
||||
func expandHomePath(path string) string {
|
||||
|
||||
@@ -204,7 +204,7 @@ var orgSeededKinds = map[string]bool{
|
||||
var orgEmptyDocuments = map[string]string{
|
||||
ChannelSettingsKind: `{"operator":"","show_tool_calls":false,"max_steps":0,"allowed_user_ids":[],"rate_limit":{"window":0,"max_requests":0},"unrestricted_filesystem":false,"workspace":"","agent":""}`,
|
||||
TaskPolicyKind: `{"notify_webhook":"","notify_min_severity":"","notify_secret":{}}`,
|
||||
SchedulingPolicyKind: `{"poll_interval":"","max_retries":0,"dispatch":{"trigger":"","ack_reaction":""}}`,
|
||||
SchedulingPolicyKind: `{"poll_interval":"","max_retries":0,"intake_per_pass":0,"dispatch":{"trigger":"","ack_reaction":""}}`,
|
||||
}
|
||||
|
||||
// resource seeds an org's workflow documents on first use, including orgs
|
||||
|
||||
@@ -27,6 +27,8 @@ const (
|
||||
type schedulingPolicy struct {
|
||||
PollInterval string `json:"poll_interval"`
|
||||
MaxRetries int `json:"max_retries"`
|
||||
// IntakePerPass caps the tasks one issue poll creates.
|
||||
IntakePerPass int `json:"intake_per_pass"`
|
||||
// Label is the dispatch trigger label. Nil keeps the file's label.
|
||||
Label *string `json:"label,omitempty"`
|
||||
Dispatch config.Dispatch `json:"dispatch"`
|
||||
@@ -46,7 +48,7 @@ func operationalDefinitions() []Definition {
|
||||
// everywhere the record reports it.
|
||||
{Kind: ChannelSettingsKind, Title: "Channel settings", ApplyMode: "restart-required", Document: channelSettings{}, Seed: seedChannels, Validate: validateChannels, Normalize: normalizeChannels},
|
||||
{Kind: SchedulingPolicyKind, Title: "Scheduling policy", ApplyMode: "live", Document: schedulingPolicy{}, Seed: func(cfg config.Config) any {
|
||||
return schedulingPolicy{PollInterval: cfg.PollInterval.Std().String(), MaxRetries: cfg.MaxRetries, Label: &cfg.Label, Dispatch: cfg.Dispatch}
|
||||
return schedulingPolicy{PollInterval: cfg.PollInterval.Std().String(), MaxRetries: cfg.MaxRetries, IntakePerPass: cfg.IntakePerPass, Label: &cfg.Label, Dispatch: cfg.Dispatch}
|
||||
}, Validate: validateScheduling},
|
||||
// Live: changed MCP servers are reconnected, rolling back to the old engine
|
||||
// if the new one fails to start.
|
||||
@@ -117,6 +119,9 @@ func validateScheduling(input []byte) error {
|
||||
if policy.MaxRetries < 0 {
|
||||
return fmt.Errorf("max_retries must not be negative")
|
||||
}
|
||||
if policy.IntakePerPass < 0 {
|
||||
return fmt.Errorf("intake_per_pass must not be negative")
|
||||
}
|
||||
if policy.PollInterval != "" {
|
||||
interval, err := time.ParseDuration(policy.PollInterval)
|
||||
if err != nil || interval <= 0 {
|
||||
@@ -150,8 +155,8 @@ func checkSchedulingWrite(orgID string, value []byte) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if policy.PollInterval != "" || policy.MaxRetries != 0 {
|
||||
return fmt.Errorf("%w: poll_interval and max_retries are instance settings; an org sets the dispatch rule and label only", ErrValidation)
|
||||
if policy.PollInterval != "" || policy.MaxRetries != 0 || policy.IntakePerPass != 0 {
|
||||
return fmt.Errorf("%w: poll_interval, max_retries and intake_per_pass are instance settings; an org sets the dispatch rule and label only", ErrValidation)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -132,6 +132,9 @@ func layerScheduling(out *config.Config, policy schedulingPolicy) error {
|
||||
if policy.MaxRetries != 0 {
|
||||
out.MaxRetries = policy.MaxRetries
|
||||
}
|
||||
if policy.IntakePerPass != 0 {
|
||||
out.IntakePerPass = policy.IntakePerPass
|
||||
}
|
||||
if policy.Dispatch.Trigger != "" {
|
||||
out.Dispatch.Trigger = policy.Dispatch.Trigger
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package worktree
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrDiskLow means the work volume stayed below the free-space floor.
|
||||
var ErrDiskLow = errors.New("disk low")
|
||||
|
||||
// FreeBytes reports the space available to unprivileged writers on the volume
|
||||
// holding the work directory, which may not exist yet.
|
||||
func (m *Manager) FreeBytes() (uint64, error) {
|
||||
path := m.WorkDir
|
||||
for {
|
||||
var st syscall.Statfs_t
|
||||
err := syscall.Statfs(path, &st)
|
||||
if err == nil {
|
||||
return st.Bavail * uint64(st.Bsize), nil //nolint:gosec // Bsize is positive
|
||||
}
|
||||
parent := filepath.Dir(path)
|
||||
if !errors.Is(err, os.ErrNotExist) || parent == path {
|
||||
return 0, err
|
||||
}
|
||||
path = parent
|
||||
}
|
||||
}
|
||||
|
||||
// AwaitSpace blocks until free reports at least floor bytes, checking every
|
||||
// interval for up to wait, so a full volume delays a clone instead of failing
|
||||
// it half-written. A volume that cannot be measured does not block the clone.
|
||||
func AwaitSpace(ctx context.Context, free func() (uint64, error), floor uint64, interval, wait time.Duration) error {
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
have, err := free()
|
||||
if err == nil && have < floor {
|
||||
if !time.Now().Add(interval).Before(deadline) {
|
||||
return fmt.Errorf("%w: %d MiB free on the work volume, %d MiB needed to clone", ErrDiskLow, have>>20, floor>>20)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(interval):
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package worktree
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAwaitSpace(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
reports []uint64
|
||||
wantErr error
|
||||
}{
|
||||
{"enough at once", []uint64{10}, nil},
|
||||
{"low then recovers", []uint64{1, 1, 10}, nil},
|
||||
{"stays low", []uint64{1}, ErrDiskLow},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
free := func() (uint64, error) {
|
||||
i := min(calls, len(tt.reports)-1)
|
||||
calls++
|
||||
return tt.reports[i], nil
|
||||
}
|
||||
err := AwaitSpace(context.Background(), free, 5, time.Millisecond, 20*time.Millisecond)
|
||||
if !errors.Is(err, tt.wantErr) {
|
||||
t.Fatalf("err = %v, want %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ const KIND = "scheduling-policy";
|
||||
interface SchedulingPolicy {
|
||||
poll_interval: string;
|
||||
max_retries: number;
|
||||
intake_per_pass: number;
|
||||
label?: string;
|
||||
dispatch: { trigger: string; ack_reaction: string };
|
||||
}
|
||||
@@ -147,6 +148,25 @@ const triggers = [
|
||||
</Field>
|
||||
</form.Field>
|
||||
|
||||
<form.Field v-slot="{ field }" name="intake_per_pass">
|
||||
<Field orientation="responsive">
|
||||
<FieldContent>
|
||||
<FieldLabel>Tasks per poll</FieldLabel>
|
||||
<FieldDescription>Most tasks one poll creates. The rest follow on later polls.</FieldDescription>
|
||||
</FieldContent>
|
||||
<div class="flex min-w-0 flex-wrap items-center gap-3 @md/field-group:flex-1">
|
||||
<NumberField :model-value="field.state.value" :min="0" class="w-32" @update:model-value="field.handleChange">
|
||||
<NumberFieldContent>
|
||||
<NumberFieldDecrement />
|
||||
<NumberFieldInput class="font-mono" aria-label="Tasks per poll" />
|
||||
<NumberFieldIncrement />
|
||||
</NumberFieldContent>
|
||||
</NumberField>
|
||||
<DraftHint :kind="KIND" path="intake_per_pass" />
|
||||
</div>
|
||||
</Field>
|
||||
</form.Field>
|
||||
|
||||
<form.Field v-slot="{ field }" name="poll_interval">
|
||||
<Field orientation="responsive" :data-invalid="fieldInvalid(field)">
|
||||
<FieldContent><FieldLabel for="sp-poll">Poll interval</FieldLabel></FieldContent>
|
||||
|
||||
Reference in new issue
Block a user