fix(daemon): a per-task stop interrupts a resumed run reacquiring capacity (#1937)
This commit is contained in:
3 files changed
+54
-6
No files matched your search
@@ -193,6 +193,17 @@ type taskHold struct {
|
||||
// task's id.
|
||||
id int64
|
||||
attempt int
|
||||
// stopped closes when the run's own context ends, so a re-acquire waiting
|
||||
// for capacity stops with a per-task stop. A context cannot be stored on the
|
||||
// hold (containedctx), hence the channel; nil until the run has one.
|
||||
stopped <-chan struct{}
|
||||
}
|
||||
|
||||
// bindStop ties a re-acquire's wait to the run's own context.
|
||||
func (h *taskHold) bindStop(ctx context.Context) {
|
||||
h.dispatcher.mu.Lock()
|
||||
h.stopped = ctx.Done()
|
||||
h.dispatcher.mu.Unlock()
|
||||
}
|
||||
|
||||
// name records which task's capacity this hold is, so a wait on a callee can
|
||||
|
||||
@@ -44,6 +44,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol
|
||||
// Register first so the task can be stopped during setup.
|
||||
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
||||
defer finished()
|
||||
hold.bindStop(ctx)
|
||||
|
||||
if d.parkIfIdentityUnresolvable(ctx, task) {
|
||||
return
|
||||
|
||||
@@ -366,12 +366,27 @@ func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan s
|
||||
// takeSlotLocked blocks until the global limit has room, then counts the run
|
||||
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
|
||||
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
|
||||
// Callers hold d.mu.
|
||||
func (d *taskDispatcher) takeSlotLocked() {
|
||||
// It returns ctx's error, taking nothing, once ctx is done. Callers hold d.mu.
|
||||
func (d *taskDispatcher) takeSlotLocked(ctx context.Context) error {
|
||||
// Broadcasting under the lock keeps the wake from slipping between the ctx
|
||||
// check below and the Wait.
|
||||
stop := context.AfterFunc(ctx, func() {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
})
|
||||
defer stop()
|
||||
for d.limit > 0 && d.active >= d.limit {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.slotsFree.Wait()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
d.active++
|
||||
return nil
|
||||
}
|
||||
|
||||
// giveBackLocked gives back everything the run holds: its slot under the
|
||||
@@ -432,8 +447,8 @@ func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits
|
||||
// and a slot under the global limit. The run keeps executing meanwhile -- a
|
||||
// wait on a callee is not a pause. A hold records one slot and one place, so a
|
||||
// re-acquire queued while another is still waiting gives back whichever of the
|
||||
// two it finds already recorded instead of taking a second. Once ctx is done
|
||||
// it stops waiting and gives back whatever the hold owns.
|
||||
// two it finds already recorded instead of taking a second. Once ctx is done,
|
||||
// or the run is stopped, it stops waiting and gives back whatever the hold owns.
|
||||
func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
||||
d.mu.Lock()
|
||||
if hold.finished || hold.waits > 0 {
|
||||
@@ -442,6 +457,18 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
stopped := hold.stopped
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
if stopped != nil {
|
||||
go func() {
|
||||
select {
|
||||
case <-stopped:
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
}
|
||||
var previous chan struct{}
|
||||
// A place another re-acquire already queued is the run's one place; joining
|
||||
// behind it would queue a second, which the task behind it would wait on
|
||||
@@ -465,7 +492,12 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
||||
}
|
||||
|
||||
d.mu.Lock()
|
||||
d.takeSlotLocked()
|
||||
if err := d.takeSlotLocked(ctx); err != nil {
|
||||
d.giveBackLocked(hold)
|
||||
d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
return
|
||||
}
|
||||
if hold.slot {
|
||||
// Another re-acquire took the run's one slot while this waited: give
|
||||
// back the slot this one just took. The place is the other's, so it is
|
||||
@@ -525,7 +557,11 @@ func (d *taskDispatcher) Submit(
|
||||
}
|
||||
}
|
||||
d.mu.Lock()
|
||||
d.takeSlotLocked()
|
||||
if err := d.takeSlotLocked(ctx); err != nil {
|
||||
d.mu.Unlock()
|
||||
d.leaveQueue()
|
||||
return
|
||||
}
|
||||
d.waiting--
|
||||
hold.slot = true
|
||||
d.mu.Unlock()
|
||||
|
||||
Reference in new issue
Block a user