fix(daemon): a per-task stop interrupts a resumed run reacquiring capacity (#1937)
This commit is contained in:
3 files changed
+54
-6
No files matched your search
@@ -193,6 +193,17 @@ type taskHold struct {
|
|||||||
// task's id.
|
// task's id.
|
||||||
id int64
|
id int64
|
||||||
attempt int
|
attempt int
|
||||||
|
// stopped closes when the run's own context ends, so a re-acquire waiting
|
||||||
|
// for capacity stops with a per-task stop. A context cannot be stored on the
|
||||||
|
// hold (containedctx), hence the channel; nil until the run has one.
|
||||||
|
stopped <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// bindStop ties a re-acquire's wait to the run's own context.
|
||||||
|
func (h *taskHold) bindStop(ctx context.Context) {
|
||||||
|
h.dispatcher.mu.Lock()
|
||||||
|
h.stopped = ctx.Done()
|
||||||
|
h.dispatcher.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// name records which task's capacity this hold is, so a wait on a callee can
|
// name records which task's capacity this hold is, so a wait on a callee can
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHol
|
|||||||
// Register first so the task can be stopped during setup.
|
// Register first so the task can be stopped during setup.
|
||||||
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
||||||
defer finished()
|
defer finished()
|
||||||
|
hold.bindStop(ctx)
|
||||||
|
|
||||||
if d.parkIfIdentityUnresolvable(ctx, task) {
|
if d.parkIfIdentityUnresolvable(ctx, task) {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -366,12 +366,27 @@ func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan s
|
|||||||
// takeSlotLocked blocks until the global limit has room, then counts the run
|
// takeSlotLocked blocks until the global limit has room, then counts the run
|
||||||
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
|
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
|
||||||
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
|
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
|
||||||
// Callers hold d.mu.
|
// It returns ctx's error, taking nothing, once ctx is done. Callers hold d.mu.
|
||||||
func (d *taskDispatcher) takeSlotLocked() {
|
func (d *taskDispatcher) takeSlotLocked(ctx context.Context) error {
|
||||||
|
// Broadcasting under the lock keeps the wake from slipping between the ctx
|
||||||
|
// check below and the Wait.
|
||||||
|
stop := context.AfterFunc(ctx, func() {
|
||||||
|
d.mu.Lock()
|
||||||
|
defer d.mu.Unlock()
|
||||||
|
d.slotsFree.Broadcast()
|
||||||
|
})
|
||||||
|
defer stop()
|
||||||
for d.limit > 0 && d.active >= d.limit {
|
for d.limit > 0 && d.active >= d.limit {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
d.slotsFree.Wait()
|
d.slotsFree.Wait()
|
||||||
}
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
d.active++
|
d.active++
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// giveBackLocked gives back everything the run holds: its slot under the
|
// giveBackLocked gives back everything the run holds: its slot under the
|
||||||
@@ -432,8 +447,8 @@ func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits
|
|||||||
// and a slot under the global limit. The run keeps executing meanwhile -- a
|
// and a slot under the global limit. The run keeps executing meanwhile -- a
|
||||||
// wait on a callee is not a pause. A hold records one slot and one place, so a
|
// wait on a callee is not a pause. A hold records one slot and one place, so a
|
||||||
// re-acquire queued while another is still waiting gives back whichever of the
|
// re-acquire queued while another is still waiting gives back whichever of the
|
||||||
// two it finds already recorded instead of taking a second. Once ctx is done
|
// two it finds already recorded instead of taking a second. Once ctx is done,
|
||||||
// it stops waiting and gives back whatever the hold owns.
|
// or the run is stopped, it stops waiting and gives back whatever the hold owns.
|
||||||
func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
if hold.finished || hold.waits > 0 {
|
if hold.finished || hold.waits > 0 {
|
||||||
@@ -442,6 +457,18 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
|||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
stopped := hold.stopped
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
if stopped != nil {
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-stopped:
|
||||||
|
cancel()
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
var previous chan struct{}
|
var previous chan struct{}
|
||||||
// A place another re-acquire already queued is the run's one place; joining
|
// A place another re-acquire already queued is the run's one place; joining
|
||||||
// behind it would queue a second, which the task behind it would wait on
|
// behind it would queue a second, which the task behind it would wait on
|
||||||
@@ -465,7 +492,12 @@ func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
d.takeSlotLocked()
|
if err := d.takeSlotLocked(ctx); err != nil {
|
||||||
|
d.giveBackLocked(hold)
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.slotsFree.Broadcast()
|
||||||
|
return
|
||||||
|
}
|
||||||
if hold.slot {
|
if hold.slot {
|
||||||
// Another re-acquire took the run's one slot while this waited: give
|
// Another re-acquire took the run's one slot while this waited: give
|
||||||
// back the slot this one just took. The place is the other's, so it is
|
// back the slot this one just took. The place is the other's, so it is
|
||||||
@@ -525,7 +557,11 @@ func (d *taskDispatcher) Submit(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
d.takeSlotLocked()
|
if err := d.takeSlotLocked(ctx); err != nil {
|
||||||
|
d.mu.Unlock()
|
||||||
|
d.leaveQueue()
|
||||||
|
return
|
||||||
|
}
|
||||||
d.waiting--
|
d.waiting--
|
||||||
hold.slot = true
|
hold.slot = true
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|||||||
Reference in new issue
Block a user