diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index d391dd607..b42353bcc 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -879,7 +879,9 @@ func (d *Daemon) botUserForTask(task *workflowtask.Task) string { // through WaitForTasks. func (d *Daemon) drainNATS(ctx context.Context) { dispatcher := d.taskDispatcher() - for ctx.Err() == nil { + // A full dispatcher leaves work in the stream and the store, not parked + // on slots where it already reads as running. + for ctx.Err() == nil && dispatcher.hasRoom() { msg, err := d.Tasks.Fetch(ctx) if err != nil && !errors.Is(err, eventbus.ErrNoMessage) { d.Log.ErrorContext(ctx, "nats fetch failed", "err", err) @@ -900,11 +902,12 @@ func (d *Daemon) drainNATS(ctx context.Context) { } } -// claimRequeued submits every task the store's claim fallback holds. It does +// claimRequeued submits tasks from the store's claim fallback while the +// dispatcher has room; the rest stay unclaimed for a later pass. It does // not go back to the stream between claims: each empty fetch waits a full poll // timeout, so a backlog left by a restart would outlast the systemd watchdog. func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) { - for ctx.Err() == nil { + for ctx.Err() == nil && dispatcher.hasRoom() { task, err := d.Store.ClaimNext(ctx) if err != nil { d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err) @@ -970,6 +973,9 @@ type taskDispatcher struct { slotsFree *sync.Cond repoTail map[string]chan struct{} pending int + // waiting counts submitted tasks that hold no slot yet, so a drain pass + // stops claiming once queued work alone fills the limit. + waiting int // holds maps a task id to the capacity its run holds, so a run waiting on // a callee can give that capacity back and take it again by task id. holds map[int64]*taskHold @@ -1186,6 +1192,7 @@ func (d *taskDispatcher) Submit( d.mu.Lock() d.pending++ + d.waiting++ d.mu.Unlock() go func() { defer d.finishRun(hold) @@ -1194,12 +1201,22 @@ func (d *taskDispatcher) Submit( } d.mu.Lock() d.takeSlotLocked() + d.waiting-- hold.slot = true d.mu.Unlock() process(ctx, task, hold) }() } +// hasRoom reports whether a newly submitted task could take a slot without +// queueing behind work already submitted. A run waiting on a callee holds no +// slot, so its callee still finds room. +func (d *taskDispatcher) hasRoom() bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.limit <= 0 || d.active+d.waiting < d.limit +} + // inFlight is how many submitted tasks have not returned yet. func (d *taskDispatcher) inFlight() int { d.mu.Lock() diff --git a/internal/daemon/dispatcher.go b/internal/daemon/dispatcher.go deleted file mode 100644 index 9d3072094..000000000 --- a/internal/daemon/dispatcher.go +++ /dev/null @@ -1 +0,0 @@ -package daemon diff --git a/internal/daemon/dispatcher_test.go b/internal/daemon/dispatcher_test.go index c2b15fa50..6f59251d1 100644 --- a/internal/daemon/dispatcher_test.go +++ b/internal/daemon/dispatcher_test.go @@ -626,3 +626,27 @@ func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) { t.Fatalf("the pass waited on the stream %d times for 5 claims, want 1", got) } } + +// A drain pass claims only what the dispatcher can run. Submit never blocks, +// so claiming past capacity marked the whole requeued backlog as running while +// the tasks sat waiting on slots. +func TestDrainPassClaimsOnlyWhatTheDispatcherCanRun(t *testing.T) { + release := make(chan struct{}) + defer close(release) + store := &queuedStore{park: func() { <-release }} + for id := range int64(10) { + store.queue = append(store.queue, &workflow.Task{ID: id + 1, Owner: "acme", Repo: "app"}) + } + cfg := config.Config{} + cfg.Containers.MaxConcurrency = 2 + d := &Daemon{ + Cfg: config.NewHolder(cfg), + Log: slog.New(slog.DiscardHandler), + Store: store, + Tasks: &countingTasks{}, + } + d.drainNATS(context.Background()) + if got := len(store.queue); got != 8 { + t.Fatalf("pass left %d tasks unclaimed, want 8", got) + } +} diff --git a/internal/daemon/identity.go b/internal/daemon/identity.go deleted file mode 100644 index 9d3072094..000000000 --- a/internal/daemon/identity.go +++ /dev/null @@ -1 +0,0 @@ -package daemon