fix(daemon): claim and fetch tasks only while the dispatcher has room #1799

Merged
sam merged 3 commits from fix/1797-claim-within-capacity into main 2026-11-08 03:53:07 +00:00
4 changed files with 44 additions and 5 deletions

No files matched your search

+20 -3
View File
@@ -879,7 +879,9 @@ func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
// through WaitForTasks.
func (d *Daemon) drainNATS(ctx context.Context) {
dispatcher := d.taskDispatcher()
for ctx.Err() == nil {
// A full dispatcher leaves work in the stream and the store, not parked
// on slots where it already reads as running.
for ctx.Err() == nil && dispatcher.hasRoom() {
msg, err := d.Tasks.Fetch(ctx)
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
d.Log.ErrorContext(ctx, "nats fetch failed", "err", err)
@@ -900,11 +902,12 @@ func (d *Daemon) drainNATS(ctx context.Context) {
}
}
// claimRequeued submits every task the store's claim fallback holds. It does
// claimRequeued submits tasks from the store's claim fallback while the
// dispatcher has room; the rest stay unclaimed for a later pass. It does
// not go back to the stream between claims: each empty fetch waits a full poll
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
for ctx.Err() == nil {
for ctx.Err() == nil && dispatcher.hasRoom() {
task, err := d.Store.ClaimNext(ctx)
if err != nil {
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
@@ -970,6 +973,9 @@ type taskDispatcher struct {
slotsFree *sync.Cond
repoTail map[string]chan struct{}
pending int
// waiting counts submitted tasks that hold no slot yet, so a drain pass
// stops claiming once queued work alone fills the limit.
waiting int
// holds maps a task id to the capacity its run holds, so a run waiting on
// a callee can give that capacity back and take it again by task id.
holds map[int64]*taskHold
@@ -1186,6 +1192,7 @@ func (d *taskDispatcher) Submit(
d.mu.Lock()
d.pending++
d.waiting++
d.mu.Unlock()
go func() {
defer d.finishRun(hold)
@@ -1194,12 +1201,22 @@ func (d *taskDispatcher) Submit(
}
d.mu.Lock()
d.takeSlotLocked()
d.waiting--
hold.slot = true
d.mu.Unlock()
process(ctx, task, hold)
}()
}
// hasRoom reports whether a newly submitted task could take a slot without
// queueing behind work already submitted. A run waiting on a callee holds no
// slot, so its callee still finds room.
func (d *taskDispatcher) hasRoom() bool {
d.mu.Lock()
defer d.mu.Unlock()
return d.limit <= 0 || d.active+d.waiting < d.limit
}
// inFlight is how many submitted tasks have not returned yet.
func (d *taskDispatcher) inFlight() int {
d.mu.Lock()
-1
View File
@@ -1 +0,0 @@
package daemon
+24
View File
@@ -626,3 +626,27 @@ func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) {
t.Fatalf("the pass waited on the stream %d times for 5 claims, want 1", got)
}
}
// A drain pass claims only what the dispatcher can run. Submit never blocks,
// so claiming past capacity marked the whole requeued backlog as running while
// the tasks sat waiting on slots.
func TestDrainPassClaimsOnlyWhatTheDispatcherCanRun(t *testing.T) {
release := make(chan struct{})
defer close(release)
store := &queuedStore{park: func() { <-release }}
for id := range int64(10) {
store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"})
}
cfg := config.Config{}
cfg.Containers.MaxConcurrency = 2
d := &Daemon{
Cfg: config.NewHolder(cfg),
Log: slog.New(slog.DiscardHandler),
Store: store,
Tasks: &countingTasks{},
}
d.drainNATS(context.Background())
if got := len(store.queue); got != 8 {
t.Fatalf("pass left %d tasks unclaimed, want 8", got)
}
}
-1
View File
@@ -1 +0,0 @@
package daemon