fix(daemon): a per-task stop interrupts a resumed run reacquiring capacity (#1937)

This commit is contained in:
sam committed 2026-11-14 03:51:03 +11:00
1 parent 0451545220
commit 7d6f22d227
3 files changed
+54 -6

No files matched your search

+11
View File
@@ -193,6 +193,17 @@ type taskHold struct {
// task's id.
id int64
attempt int
// stopped closes when the run's own context ends, so a re-acquire waiting
// for capacity stops with a per-task stop. A context cannot be stored on the
// hold (containedctx), hence the channel; nil until the run has one.
stopped <-chan struct{}
}
// bindStop ties a re-acquire's wait to the run's own context.
func (h *taskHold) bindStop(ctx context.Context) {
h.dispatcher.mu.Lock()
h.stopped = ctx.Done()
h.dispatcher.mu.Unlock()
}
// name records which task's capacity this hold is, so a wait on a callee can
+1
View File
@@ -44,6 +44,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol
// Register first so the task can be stopped during setup.
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
defer finished()
hold.bindStop(ctx)
if d.parkIfIdentityUnresolvable(ctx, task) {
return
+42 -6
View File
@@ -366,12 +366,27 @@ func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan s
// takeSlotLocked blocks until the global limit has room, then counts the run
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
// Callers hold d.mu.
func (d *taskDispatcher) takeSlotLocked() {
// It returns ctx's error, taking nothing, once ctx is done. Callers hold d.mu.
func (d *taskDispatcher) takeSlotLocked(ctx context.Context) error {
// Broadcasting under the lock keeps the wake from slipping between the ctx
// check below and the Wait.
stop := context.AfterFunc(ctx, func() {
d.mu.Lock()
defer d.mu.Unlock()
d.slotsFree.Broadcast()
})
defer stop()
for d.limit > 0 && d.active >= d.limit {
if err := ctx.Err(); err != nil {
return err
}
d.slotsFree.Wait()
}
if err := ctx.Err(); err != nil {
return err
}
d.active++
return nil
}
// giveBackLocked gives back everything the run holds: its slot under the
@@ -432,8 +447,8 @@ func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits
// and a slot under the global limit. The run keeps executing meanwhile -- a
// wait on a callee is not a pause. A hold records one slot and one place, so a
// re-acquire queued while another is still waiting gives back whichever of the
// two it finds already recorded instead of taking a second. Once ctx is done
// it stops waiting and gives back whatever the hold owns.
// two it finds already recorded instead of taking a second. Once ctx is done,
// or the run is stopped, it stops waiting and gives back whatever the hold owns.
func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
d.mu.Lock()
if hold.finished || hold.waits > 0 {
@@ -442,6 +457,18 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
d.mu.Unlock()
return
}
stopped := hold.stopped
ctx, cancel := context.WithCancel(ctx)
defer cancel()
if stopped != nil {
go func() {
select {
case <-stopped:
cancel()
case <-ctx.Done():
}
}()
}
var previous chan struct{}
// A place another re-acquire already queued is the run's one place; joining
// behind it would queue a second, which the task behind it would wait on
@@ -465,7 +492,12 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
}
d.mu.Lock()
d.takeSlotLocked()
if err := d.takeSlotLocked(ctx); err != nil {
d.giveBackLocked(hold)
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
if hold.slot {
// Another re-acquire took the run's one slot while this waited: give
// back the slot this one just took. The place is the other's, so it is
@@ -525,7 +557,11 @@ func (d *taskDispatcher) Submit(
}
}
d.mu.Lock()
d.takeSlotLocked()
if err := d.takeSlotLocked(ctx); err != nil {
d.mu.Unlock()
d.leaveQueue()
return
}
d.waiting--
hold.slot = true
d.mu.Unlock()