|
|
|
@@ -879,7 +879,9 @@ func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
|
|
|
|
|
// through WaitForTasks.
|
|
|
|
|
func (d *Daemon) drainNATS(ctx context.Context) {
|
|
|
|
|
dispatcher := d.taskDispatcher()
|
|
|
|
|
for ctx.Err() == nil {
|
|
|
|
|
// A full dispatcher leaves work in the stream and the store, not parked
|
|
|
|
|
// on slots where it already reads as running.
|
|
|
|
|
for ctx.Err() == nil && dispatcher.hasRoom() {
|
|
|
|
|
msg, err := d.Tasks.Fetch(ctx)
|
|
|
|
|
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
|
|
|
|
|
d.Log.ErrorContext(ctx, "nats fetch failed", "err", err)
|
|
|
|
@@ -900,11 +902,12 @@ func (d *Daemon) drainNATS(ctx context.Context) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// claimRequeued submits every task the store's claim fallback holds. It does
|
|
|
|
|
// claimRequeued submits tasks from the store's claim fallback while the
|
|
|
|
|
// dispatcher has room; the rest stay unclaimed for a later pass. It does
|
|
|
|
|
// not go back to the stream between claims: each empty fetch waits a full poll
|
|
|
|
|
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
|
|
|
|
|
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
|
|
|
|
|
for ctx.Err() == nil {
|
|
|
|
|
for ctx.Err() == nil && dispatcher.hasRoom() {
|
|
|
|
|
task, err := d.Store.ClaimNext(ctx)
|
|
|
|
|
if err != nil {
|
|
|
|
|
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
|
|
|
|
@@ -970,6 +973,9 @@ type taskDispatcher struct {
|
|
|
|
|
slotsFree *sync.Cond
|
|
|
|
|
repoTail map[string]chan struct{}
|
|
|
|
|
pending int
|
|
|
|
|
// waiting counts submitted tasks that hold no slot yet, so a drain pass
|
|
|
|
|
// stops claiming once queued work alone fills the limit.
|
|
|
|
|
waiting int
|
|
|
|
|
// holds maps a task id to the capacity its run holds, so a run waiting on
|
|
|
|
|
// a callee can give that capacity back and take it again by task id.
|
|
|
|
|
holds map[int64]*taskHold
|
|
|
|
@@ -1186,6 +1192,7 @@ func (d *taskDispatcher) Submit(
|
|
|
|
|
|
|
|
|
|
d.mu.Lock()
|
|
|
|
|
d.pending++
|
|
|
|
|
d.waiting++
|
|
|
|
|
d.mu.Unlock()
|
|
|
|
|
go func() {
|
|
|
|
|
defer d.finishRun(hold)
|
|
|
|
@@ -1194,12 +1201,22 @@ func (d *taskDispatcher) Submit(
|
|
|
|
|
}
|
|
|
|
|
d.mu.Lock()
|
|
|
|
|
d.takeSlotLocked()
|
|
|
|
|
d.waiting--
|
|
|
|
|
hold.slot = true
|
|
|
|
|
d.mu.Unlock()
|
|
|
|
|
process(ctx, task, hold)
|
|
|
|
|
}()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// hasRoom reports whether a newly submitted task could take a slot without
|
|
|
|
|
// queueing behind work already submitted. A run waiting on a callee holds no
|
|
|
|
|
// slot, so its callee still finds room.
|
|
|
|
|
func (d *taskDispatcher) hasRoom() bool {
|
|
|
|
|
d.mu.Lock()
|
|
|
|
|
defer d.mu.Unlock()
|
|
|
|
|
return d.limit <= 0 || d.active+d.waiting < d.limit
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// inFlight is how many submitted tasks have not returned yet.
|
|
|
|
|
func (d *taskDispatcher) inFlight() int {
|
|
|
|
|
d.mu.Lock()
|
|
|
|
|