diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 10e4e5f5b..da1c16894 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -193,6 +193,17 @@ type taskHold struct { // task's id. id int64 attempt int + // stopped closes when the run's own context ends, so a re-acquire waiting + // for capacity stops with a per-task stop. A context cannot be stored on the + // hold (containedctx), hence the channel; nil until the run has one. + stopped <-chan struct{} +} + +// bindStop ties a re-acquire's wait to the run's own context. +func (h *taskHold) bindStop(ctx context.Context) { + h.dispatcher.mu.Lock() + h.stopped = ctx.Done() + h.dispatcher.mu.Unlock() } // name records which task's capacity this hold is, so a wait on a callee can diff --git a/internal/daemon/pipeline.go b/internal/daemon/pipeline.go index ada8b5075..18b7a5db7 100644 --- a/internal/daemon/pipeline.go +++ b/internal/daemon/pipeline.go @@ -44,6 +44,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol // Register first so the task can be stopped during setup. ctx, finished := d.running.begin(ctx, task.ID, task.Identity) defer finished() + hold.bindStop(ctx) if d.parkIfIdentityUnresolvable(ctx, task) { return diff --git a/internal/daemon/task_bus.go b/internal/daemon/task_bus.go index c497b2e48..d1765ca4e 100644 --- a/internal/daemon/task_bus.go +++ b/internal/daemon/task_bus.go @@ -366,12 +366,27 @@ func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan s // takeSlotLocked blocks until the global limit has room, then counts the run // in. limit <= 0 means unlimited. The limit is re-read on every wake, so // SetMaxConcurrency reaches waiting tasks without recreating the dispatcher. -// Callers hold d.mu. -func (d *taskDispatcher) takeSlotLocked() { +// It returns ctx's error, taking nothing, once ctx is done. Callers hold d.mu. +func (d *taskDispatcher) takeSlotLocked(ctx context.Context) error { + // Broadcasting under the lock keeps the wake from slipping between the ctx + // check below and the Wait. + stop := context.AfterFunc(ctx, func() { + d.mu.Lock() + defer d.mu.Unlock() + d.slotsFree.Broadcast() + }) + defer stop() for d.limit > 0 && d.active >= d.limit { + if err := ctx.Err(); err != nil { + return err + } d.slotsFree.Wait() } + if err := ctx.Err(); err != nil { + return err + } d.active++ + return nil } // giveBackLocked gives back everything the run holds: its slot under the @@ -432,8 +447,8 @@ func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits // and a slot under the global limit. The run keeps executing meanwhile -- a // wait on a callee is not a pause. A hold records one slot and one place, so a // re-acquire queued while another is still waiting gives back whichever of the -// two it finds already recorded instead of taking a second. Once ctx is done -// it stops waiting and gives back whatever the hold owns. +// two it finds already recorded instead of taking a second. Once ctx is done, +// or the run is stopped, it stops waiting and gives back whatever the hold owns. func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) { d.mu.Lock() if hold.finished || hold.waits > 0 { @@ -442,6 +457,18 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) { d.mu.Unlock() return } + stopped := hold.stopped + ctx, cancel := context.WithCancel(ctx) + defer cancel() + if stopped != nil { + go func() { + select { + case <-stopped: + cancel() + case <-ctx.Done(): + } + }() + } var previous chan struct{} // A place another re-acquire already queued is the run's one place; joining // behind it would queue a second, which the task behind it would wait on @@ -465,7 +492,12 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) { } d.mu.Lock() - d.takeSlotLocked() + if err := d.takeSlotLocked(ctx); err != nil { + d.giveBackLocked(hold) + d.mu.Unlock() + d.slotsFree.Broadcast() + return + } if hold.slot { // Another re-acquire took the run's one slot while this waited: give // back the slot this one just took. The place is the other's, so it is @@ -525,7 +557,11 @@ func (d *taskDispatcher) Submit( } } d.mu.Lock() - d.takeSlotLocked() + if err := d.takeSlotLocked(ctx); err != nil { + d.mu.Unlock() + d.leaveQueue() + return + } d.waiting-- hold.slot = true d.mu.Unlock()