602 lines
19 KiB
Go
602 lines
19 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/identity"
|
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
|
"github.com/samcharles93/archie-core/internal/eventbus"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/natsrpc"
|
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
|
)
|
|
|
|
// TaskBus takes work from the NATS stream and the store's requeue fallback,
|
|
// runs it under one global dispatcher, and publishes discovered work.
|
|
type TaskBus struct {
|
|
transport TaskTransport
|
|
store storecontract.TaskStore
|
|
identities identityLookup
|
|
log *slog.Logger
|
|
orgs orgResolver
|
|
process func(context.Context, *workflow.Task, *taskHold)
|
|
emit func(context.Context, events.Event)
|
|
markPass func(loop string)
|
|
dispatcher *taskDispatcher
|
|
}
|
|
|
|
// TaskBusDeps are the inputs a TaskBus runs on. Every field is required.
|
|
type TaskBusDeps struct {
|
|
Transport TaskTransport
|
|
Store storecontract.TaskStore
|
|
Identities identityLookup
|
|
Log *slog.Logger
|
|
Orgs orgResolver
|
|
MaxConcurrency int
|
|
AllowConcurrent func(*workflow.Task) bool
|
|
Process func(context.Context, *workflow.Task, *taskHold)
|
|
Emit func(context.Context, events.Event)
|
|
MarkPass func(loop string)
|
|
}
|
|
|
|
// NewTaskBus builds a bus whose dispatcher starts at MaxConcurrency.
|
|
func NewTaskBus(deps TaskBusDeps) *TaskBus {
|
|
return &TaskBus{
|
|
transport: deps.Transport,
|
|
store: deps.Store,
|
|
identities: deps.Identities,
|
|
log: deps.Log,
|
|
orgs: deps.Orgs,
|
|
process: deps.Process,
|
|
emit: deps.Emit,
|
|
markPass: deps.MarkPass,
|
|
dispatcher: newTaskDispatcher(deps.MaxConcurrency, deps.AllowConcurrent),
|
|
}
|
|
}
|
|
|
|
// WireTaskBus builds the daemon's task bus from its fields. The composition
|
|
// root calls it once the fields are set, and before the daemon is published:
|
|
// process is unexported, so only this package can name it.
|
|
func (d *Daemon) WireTaskBus() {
|
|
d.TaskBus = NewTaskBus(TaskBusDeps{
|
|
Transport: d.TaskTransport,
|
|
Store: d.Store,
|
|
Identities: d.Identity,
|
|
Log: d.Log,
|
|
Orgs: d.Identity,
|
|
MaxConcurrency: d.Cfg.Get().Containers.MaxConcurrency,
|
|
AllowConcurrent: d.allowConcurrentForTask,
|
|
Process: d.process,
|
|
Emit: d.emit,
|
|
MarkPass: d.markPass,
|
|
})
|
|
}
|
|
|
|
// identityLookup reads one identity record.
|
|
type identityLookup interface {
|
|
Lookup(ctx context.Context, id identity.IdentityID) (identity.Identity, error)
|
|
}
|
|
|
|
// drainLoop is the only task drainer and capture dispatcher while Run is
|
|
// active. It runs apart from the poll tick so a captured event becomes a
|
|
// running task within about a second regardless of poll_interval or forge
|
|
// latency. Submit never blocks, so a full dispatcher
|
|
// just leaves tasks waiting on slots and the loop stays paced by the ticker.
|
|
func (b *TaskBus) drainLoop(ctx context.Context, dispatchBindings func(context.Context)) {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
b.markPass("drain")
|
|
dispatchBindings(ctx)
|
|
b.drainNATS(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for
|
|
// requeued tasks (waiting_approval approval, retry-parked) that didn't come
|
|
// through a NATS publish. It returns once the stream and the claim fallback
|
|
// are both empty: in-flight tasks keep running, so a task waiting on a callee
|
|
// does not hold up the pass that would claim it. Shutdown waits for them
|
|
// through WaitForTasks.
|
|
func (b *TaskBus) drainNATS(ctx context.Context) {
|
|
// A full dispatcher leaves work in the stream and the store, not parked
|
|
// on slots where it already reads as running.
|
|
for ctx.Err() == nil && b.dispatcher.hasRoom() {
|
|
msg, err := b.transport.Fetch(ctx)
|
|
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
|
|
b.log.ErrorContext(ctx, "nats fetch failed", "err", err)
|
|
break
|
|
}
|
|
if err == nil {
|
|
b.submitNATSTask(ctx, msg)
|
|
b.markPass("drain")
|
|
continue
|
|
}
|
|
|
|
// ErrNoMessage means the stream is drained. That is precisely when
|
|
// the SQLite fallback must run: requeued tasks (waiting_approval
|
|
// approval, retry-parked) never went through a NATS publish, so
|
|
// nothing will ever arrive on the stream to trigger them.
|
|
b.claimRequeued(ctx)
|
|
return
|
|
}
|
|
}
|
|
|
|
// claimRequeued submits tasks from the store's claim fallback while the
|
|
// dispatcher has room; the rest stay unclaimed for a later pass. It does
|
|
// not go back to the stream between claims: each empty fetch waits a full poll
|
|
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
|
|
func (b *TaskBus) claimRequeued(ctx context.Context) {
|
|
for ctx.Err() == nil && b.dispatcher.hasRoom() {
|
|
task, err := b.store.ClaimNext(ctx)
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "sqlite claim failed", "err", err)
|
|
return
|
|
}
|
|
if task == nil {
|
|
return
|
|
}
|
|
b.dispatcher.Submit(ctx, task, b.process)
|
|
b.markPass("drain")
|
|
}
|
|
}
|
|
|
|
// WaitForTasks blocks until every submitted task's process has returned. A
|
|
// drain pass no longer waits for its own tasks, so shutdown is where in-flight
|
|
// work is waited for.
|
|
func (b *TaskBus) WaitForTasks() {
|
|
b.dispatcher.Wait()
|
|
}
|
|
|
|
// ResizeTaskDispatcher applies a new containers.max_concurrency to the
|
|
// running global task dispatcher without a restart. Running tasks keep their
|
|
// slots; a lowered limit is honoured as they finish, and a raised limit
|
|
// admits queued work immediately.
|
|
func (b *TaskBus) ResizeTaskDispatcher(maxConcurrency int) {
|
|
b.dispatcher.SetMaxConcurrency(maxConcurrency)
|
|
}
|
|
|
|
// submitNATSTask decodes a fetched message and queues it for processing. A
|
|
// message that cannot be decoded is acked rather than redelivered: it will
|
|
// never decode on a retry, so leaving it unacked would block the queue.
|
|
func (b *TaskBus) submitNATSTask(ctx context.Context, msg eventbus.Message) {
|
|
envelope, err := workintake.DecodeTask(msg.Data())
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err)
|
|
if err := msg.Ack(); err != nil {
|
|
b.log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err)
|
|
}
|
|
return
|
|
}
|
|
task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
|
|
b.dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) {
|
|
b.processNATSTask(ctx, msg, hold)
|
|
})
|
|
}
|
|
|
|
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
|
|
// and runs the workflow. The message is ack'd on terminal (park is a valid
|
|
// outcome). Nak on transient errors so NATS redelivers.
|
|
func (b *TaskBus) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) {
|
|
ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header())
|
|
defer span.End()
|
|
tm, err := workintake.DecodeTask(msg.Data())
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "nats decode failed", "err", err)
|
|
if err := msg.Ack(); err != nil {
|
|
b.log.WarnContext(ctx, "ack failed", "err", err)
|
|
}
|
|
// bad message, don't retry
|
|
return
|
|
}
|
|
|
|
if !b.acceptTaskPublisher(ctx, msg, tm.Identity) {
|
|
return
|
|
}
|
|
|
|
inserted, err := b.store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "nats enqueue failed", "err", err)
|
|
if err := msg.Nak(); err != nil {
|
|
b.log.WarnContext(ctx, "nats nak failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
if !inserted {
|
|
// Already tracked -- dedup. Ack and move on.
|
|
if err := msg.Ack(); err != nil {
|
|
b.log.WarnContext(ctx, "ack failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
task, err := b.store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
|
|
if err != nil {
|
|
b.log.ErrorContext(ctx, "nats claim failed", "err", err)
|
|
if err := msg.Nak(); err != nil {
|
|
b.log.WarnContext(ctx, "nats nak failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
if task == nil {
|
|
// Claimed by another consumer, or task is not queued. Ack.
|
|
if err := msg.Ack(); err != nil {
|
|
b.log.WarnContext(ctx, "ack failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
b.emitTaskQueued(ctx, task)
|
|
b.process(ctx, task, hold)
|
|
if err := msg.Ack(); err != nil {
|
|
b.log.WarnContext(ctx, "ack failed", "err", err)
|
|
}
|
|
}
|
|
|
|
func (b *TaskBus) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool {
|
|
if ok, reason, class := b.orgs.identityMayAct(ctx, name); !ok {
|
|
b.log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject())
|
|
if class == taskstate.ParkTransient {
|
|
if err := msg.Nak(); err != nil {
|
|
b.log.WarnContext(ctx, "nak failed", "err", err)
|
|
}
|
|
return false
|
|
}
|
|
if err := msg.Ack(); err != nil {
|
|
b.log.WarnContext(ctx, "ack failed", "err", err)
|
|
}
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// emitTaskQueued records the queued task under the identity that polled it,
|
|
// which is both the actor and the authority the task was created with.
|
|
func (b *TaskBus) emitTaskQueued(ctx context.Context, task *workflow.Task) {
|
|
e := events.Event{
|
|
Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo,
|
|
Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt,
|
|
ActorID: task.Identity, PrincipalID: task.Identity,
|
|
}
|
|
if value, err := b.identities.Lookup(ctx, identity.IdentityID(task.Identity)); err == nil {
|
|
e.ActorKind = string(value.Kind)
|
|
}
|
|
b.emit(ctx, e)
|
|
}
|
|
|
|
// PublishTask validates a task envelope, stamps the producing identity's
|
|
// org and publishes it. The idempotency key dedups rediscovered issues.
|
|
func (b *TaskBus) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
|
|
if task.Identity == "" {
|
|
return fmt.Errorf("publish task %s: no identity", task.Ref())
|
|
}
|
|
resolved, err := b.orgs.identityOrg(ctx, identity.IdentityID(task.Identity))
|
|
if err != nil {
|
|
return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err)
|
|
}
|
|
task.Org = resolved
|
|
payload, err := task.Encode()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return b.transport.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
|
|
}
|
|
|
|
// PublishUnique publishes a raw message on the bus, deduplicated by key.
|
|
func (b *TaskBus) PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error {
|
|
return b.transport.PublishUnique(ctx, subject, idempotencyKey, payload)
|
|
}
|
|
|
|
// Request sends a request on the bus and waits for its reply.
|
|
func (b *TaskBus) Request(ctx context.Context, subject string, payload []byte) ([]byte, error) {
|
|
return b.transport.Request(ctx, subject, payload)
|
|
}
|
|
|
|
// InFlight counts the same submitted work that graceful shutdown waits for,
|
|
// including tasks waiting for a repository or concurrency slot.
|
|
func (b *TaskBus) InFlight() int {
|
|
return b.dispatcher.inFlight()
|
|
}
|
|
|
|
// taskDispatcher limits concurrent tasks globally and runs one task per repo
|
|
// at a time, in submit order, unless the repo allows concurrency.
|
|
type taskDispatcher struct {
|
|
allowConcurrent func(task *workflow.Task) bool
|
|
|
|
mu sync.Mutex
|
|
limit int
|
|
active int
|
|
slotsFree *sync.Cond
|
|
repoTail map[string]chan struct{}
|
|
pending int
|
|
// waiting counts submitted tasks that hold no slot yet, so a drain pass
|
|
// stops claiming once queued work alone fills the limit.
|
|
waiting int
|
|
// holds maps a task id to the capacity its run holds, so a run waiting on
|
|
// a callee can give that capacity back and take it again by task id.
|
|
holds map[int64]*taskHold
|
|
}
|
|
|
|
// SetMaxConcurrency accepts a new global limit without a restart. Running
|
|
// tasks keep their slots; a lowered limit is enforced as they finish, and a
|
|
// raised limit admits queued work immediately.
|
|
func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) {
|
|
d.mu.Lock()
|
|
d.limit = maxConcurrency
|
|
d.mu.Unlock()
|
|
d.slotsFree.Broadcast()
|
|
}
|
|
|
|
func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher {
|
|
if allowConcurrent == nil {
|
|
allowConcurrent = func(*workflow.Task) bool { return false }
|
|
}
|
|
d := &taskDispatcher{
|
|
limit: maxConcurrency,
|
|
allowConcurrent: allowConcurrent,
|
|
repoTail: make(map[string]chan struct{}),
|
|
holds: make(map[int64]*taskHold),
|
|
}
|
|
d.slotsFree = sync.NewCond(&d.mu)
|
|
return d
|
|
}
|
|
|
|
// joinRepoLocked appends baton to the tail of repo's chain and returns the
|
|
// baton ahead of it, which the joining task is blocked on. Callers hold d.mu.
|
|
func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} {
|
|
previous := d.repoTail[repo]
|
|
d.repoTail[repo] = baton
|
|
return previous
|
|
}
|
|
|
|
// takeSlotLocked blocks until the global limit has room, then counts the run
|
|
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
|
|
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
|
|
// It returns ctx's error, taking nothing, once ctx is done. Callers hold d.mu.
|
|
func (d *taskDispatcher) takeSlotLocked(ctx context.Context) error {
|
|
// Broadcasting under the lock keeps the wake from slipping between the ctx
|
|
// check below and the Wait.
|
|
stop := context.AfterFunc(ctx, func() {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.slotsFree.Broadcast()
|
|
})
|
|
defer stop()
|
|
for d.limit > 0 && d.active >= d.limit {
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
d.slotsFree.Wait()
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
d.active++
|
|
return nil
|
|
}
|
|
|
|
// giveBackLocked gives back everything the run holds: its slot under the
|
|
// global limit, and its place in its repo's chain, which releases the task
|
|
// queued behind it. Callers hold d.mu.
|
|
func (d *taskDispatcher) giveBackLocked(hold *taskHold) {
|
|
if hold.slot {
|
|
d.active--
|
|
hold.slot = false
|
|
}
|
|
if hold.baton == nil {
|
|
return
|
|
}
|
|
close(hold.baton)
|
|
if d.repoTail[hold.repo] == hold.baton {
|
|
delete(d.repoTail, hold.repo)
|
|
}
|
|
hold.baton = nil
|
|
}
|
|
|
|
// finishRun retires a returned run: it holds no capacity any more, and no
|
|
// Resume may take any for it. Whatever an in-flight resume takes later is
|
|
// given back by reacquire, which sees finished.
|
|
func (d *taskDispatcher) finishRun(hold *taskHold) {
|
|
d.mu.Lock()
|
|
hold.finished = true
|
|
if hold.id != 0 && d.holds[hold.id] == hold {
|
|
delete(d.holds, hold.id)
|
|
}
|
|
d.giveBackLocked(hold)
|
|
d.pending--
|
|
d.mu.Unlock()
|
|
d.slotsFree.Broadcast()
|
|
}
|
|
|
|
// setCallWaits applies a durable snapshot without accumulating duplicate events.
|
|
func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits int) {
|
|
d.mu.Lock()
|
|
if d.holds[hold.id] != hold || hold.finished {
|
|
d.mu.Unlock()
|
|
return
|
|
}
|
|
previous := hold.waits
|
|
hold.waits = waits
|
|
if waits > 0 {
|
|
d.giveBackLocked(hold)
|
|
}
|
|
d.mu.Unlock()
|
|
if waits > 0 {
|
|
d.slotsFree.Broadcast()
|
|
} else if previous > 0 {
|
|
// Reacquiring may block behind another task; the caller keeps running.
|
|
go d.reacquire(ctx, hold)
|
|
}
|
|
}
|
|
|
|
// reacquire returns a resumed run's capacity: its place in its repo's chain
|
|
// and a slot under the global limit. The run keeps executing meanwhile -- a
|
|
// wait on a callee is not a pause. A hold records one slot and one place, so a
|
|
// re-acquire queued while another is still waiting gives back whichever of the
|
|
// two it finds already recorded instead of taking a second. Once ctx is done,
|
|
// or the run is stopped, it stops waiting and gives back whatever the hold owns.
|
|
func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
|
|
d.mu.Lock()
|
|
if hold.finished || hold.waits > 0 {
|
|
// The run returned, or started waiting again, after the reconciliation that
|
|
// queued this: it needs no capacity.
|
|
d.mu.Unlock()
|
|
return
|
|
}
|
|
stopped := hold.stopped
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
if stopped != nil {
|
|
go func() {
|
|
select {
|
|
case <-stopped:
|
|
cancel()
|
|
case <-ctx.Done():
|
|
}
|
|
}()
|
|
}
|
|
var previous chan struct{}
|
|
// A place another re-acquire already queued is the run's one place; joining
|
|
// behind it would queue a second, which the task behind it would wait on
|
|
// forever.
|
|
if hold.chain && hold.baton == nil {
|
|
hold.baton = make(chan struct{})
|
|
previous = d.joinRepoLocked(hold.repo, hold.baton)
|
|
}
|
|
d.mu.Unlock()
|
|
|
|
if previous != nil {
|
|
select {
|
|
case <-previous:
|
|
case <-ctx.Done():
|
|
d.mu.Lock()
|
|
d.giveBackLocked(hold)
|
|
d.mu.Unlock()
|
|
d.slotsFree.Broadcast()
|
|
return
|
|
}
|
|
}
|
|
|
|
d.mu.Lock()
|
|
if err := d.takeSlotLocked(ctx); err != nil {
|
|
d.giveBackLocked(hold)
|
|
d.mu.Unlock()
|
|
d.slotsFree.Broadcast()
|
|
return
|
|
}
|
|
if hold.slot {
|
|
// Another re-acquire took the run's one slot while this waited: give
|
|
// back the slot this one just took. The place is the other's, so it is
|
|
// left alone.
|
|
d.active--
|
|
d.mu.Unlock()
|
|
d.slotsFree.Broadcast()
|
|
return
|
|
}
|
|
hold.slot = true
|
|
if hold.finished || hold.waits > 0 {
|
|
// The run ended, or started waiting again, while this was queued for
|
|
// capacity: give back what it took, so no slot or place in the repo
|
|
// chain leaks.
|
|
d.giveBackLocked(hold)
|
|
d.mu.Unlock()
|
|
d.slotsFree.Broadcast()
|
|
return
|
|
}
|
|
d.mu.Unlock()
|
|
}
|
|
|
|
// Submit queues a task's process behind the global limit and, unless the repo
|
|
// allows concurrency, behind the task already holding that repo. process is
|
|
// handed the run's capacity so it can name it once it knows the task's id and
|
|
// give it back while it waits on a callee. A run whose ctx ends while it waits
|
|
// for its repo's place returns without running process.
|
|
func (d *taskDispatcher) Submit(
|
|
ctx context.Context,
|
|
task *workflow.Task,
|
|
process func(context.Context, *workflow.Task, *taskHold),
|
|
) {
|
|
repo := task.Owner + "/" + task.Repo
|
|
hold := &taskHold{dispatcher: d, repo: repo}
|
|
|
|
var previous chan struct{}
|
|
if !d.allowConcurrent(task) {
|
|
hold.chain = true
|
|
hold.baton = make(chan struct{})
|
|
d.mu.Lock()
|
|
previous = d.joinRepoLocked(repo, hold.baton)
|
|
d.mu.Unlock()
|
|
}
|
|
|
|
d.mu.Lock()
|
|
d.pending++
|
|
d.waiting++
|
|
d.mu.Unlock()
|
|
go func() {
|
|
defer d.finishRun(hold)
|
|
if previous != nil {
|
|
select {
|
|
case <-previous:
|
|
case <-ctx.Done():
|
|
d.leaveQueue()
|
|
return
|
|
}
|
|
}
|
|
d.mu.Lock()
|
|
if err := d.takeSlotLocked(ctx); err != nil {
|
|
d.mu.Unlock()
|
|
d.leaveQueue()
|
|
return
|
|
}
|
|
d.waiting--
|
|
hold.slot = true
|
|
d.mu.Unlock()
|
|
process(ctx, task, hold)
|
|
}()
|
|
}
|
|
|
|
// leaveQueue drops a submitted run that was cancelled before it took a slot.
|
|
func (d *taskDispatcher) leaveQueue() {
|
|
d.mu.Lock()
|
|
d.waiting--
|
|
d.mu.Unlock()
|
|
}
|
|
|
|
// hasRoom reports whether a newly submitted task could take a slot without
|
|
// queueing behind work already submitted. A run waiting on a callee holds no
|
|
// slot, so its callee still finds room.
|
|
func (d *taskDispatcher) hasRoom() bool {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
return d.limit <= 0 || d.active+d.waiting < d.limit
|
|
}
|
|
|
|
// inFlight is how many submitted tasks have not returned yet.
|
|
func (d *taskDispatcher) inFlight() int {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
return d.pending
|
|
}
|
|
|
|
func (d *taskDispatcher) Wait() {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
for d.pending > 0 {
|
|
d.slotsFree.Wait()
|
|
}
|
|
}
|