package daemon import ( "context" "errors" "fmt" "log/slog" "strings" "sync" "time" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/natsrpc" "github.com/samcharles93/archie-core/internal/taskstate" ) // TaskBus takes work from the NATS stream and the store's requeue fallback, // runs it under one global dispatcher, and publishes discovered work. type TaskBus struct { transport TaskTransport store storecontract.TaskStore identities identityLookup log *slog.Logger orgs orgResolver process func(context.Context, *workflow.Task, *taskHold) emit func(context.Context, events.Event) markPass func(loop string) dispatcher *taskDispatcher } // TaskBusDeps are the inputs a TaskBus runs on. Every field is required. type TaskBusDeps struct { Transport TaskTransport Store storecontract.TaskStore Identities identityLookup Log *slog.Logger Orgs orgResolver MaxConcurrency int AllowConcurrent func(*workflow.Task) bool Process func(context.Context, *workflow.Task, *taskHold) Emit func(context.Context, events.Event) MarkPass func(loop string) } // NewTaskBus builds a bus whose dispatcher starts at MaxConcurrency. func NewTaskBus(deps TaskBusDeps) *TaskBus { return &TaskBus{ transport: deps.Transport, store: deps.Store, identities: deps.Identities, log: deps.Log, orgs: deps.Orgs, process: deps.Process, emit: deps.Emit, markPass: deps.MarkPass, dispatcher: newTaskDispatcher(deps.MaxConcurrency, deps.AllowConcurrent), } } // WireTaskBus builds the daemon's task bus from its fields. The composition // root calls it once the fields are set, and before the daemon is published: // process is unexported, so only this package can name it. func (d *Daemon) WireTaskBus() { d.TaskBus = NewTaskBus(TaskBusDeps{ Transport: d.TaskTransport, Store: d.Store, Identities: d.Identity, Log: d.Log, Orgs: d.Identity, MaxConcurrency: d.Cfg.Get().Containers.MaxConcurrency, AllowConcurrent: d.allowConcurrentForTask, Process: d.process, Emit: d.emit, MarkPass: d.markPass, }) } // identityLookup reads one identity record. type identityLookup interface { Lookup(ctx context.Context, id identity.IdentityID) (identity.Identity, error) } // drainLoop is the only task drainer and capture dispatcher while Run is // active. It runs apart from the poll tick so a captured event becomes a // running task within about a second regardless of poll_interval or forge // latency. Submit never blocks, so a full dispatcher // just leaves tasks waiting on slots and the loop stays paced by the ticker. func (b *TaskBus) drainLoop(ctx context.Context, dispatchBindings func(context.Context)) { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { b.markPass("drain") dispatchBindings(ctx) b.drainNATS(ctx) select { case <-ctx.Done(): return case <-ticker.C: } } } // drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for // requeued tasks (waiting_approval approval, retry-parked) that didn't come // through a NATS publish. It returns once the stream and the claim fallback // are both empty: in-flight tasks keep running, so a task waiting on a callee // does not hold up the pass that would claim it. Shutdown waits for them // through WaitForTasks. func (b *TaskBus) drainNATS(ctx context.Context) { // A full dispatcher leaves work in the stream and the store, not parked // on slots where it already reads as running. for ctx.Err() == nil && b.dispatcher.hasRoom() { msg, err := b.transport.Fetch(ctx) if err != nil && !errors.Is(err, eventbus.ErrNoMessage) { b.log.ErrorContext(ctx, "nats fetch failed", "err", err) break } if err == nil { b.submitNATSTask(ctx, msg) b.markPass("drain") continue } // ErrNoMessage means the stream is drained. That is precisely when // the SQLite fallback must run: requeued tasks (waiting_approval // approval, retry-parked) never went through a NATS publish, so // nothing will ever arrive on the stream to trigger them. b.claimRequeued(ctx) return } } // claimRequeued submits tasks from the store's claim fallback while the // dispatcher has room; the rest stay unclaimed for a later pass. It does // not go back to the stream between claims: each empty fetch waits a full poll // timeout, so a backlog left by a restart would outlast the systemd watchdog. func (b *TaskBus) claimRequeued(ctx context.Context) { for ctx.Err() == nil && b.dispatcher.hasRoom() { task, err := b.store.ClaimNext(ctx) if err != nil { b.log.ErrorContext(ctx, "sqlite claim failed", "err", err) return } if task == nil { return } b.dispatcher.Submit(ctx, task, b.process) b.markPass("drain") } } // WaitForTasks blocks until every submitted task's process has returned. A // drain pass no longer waits for its own tasks, so shutdown is where in-flight // work is waited for. func (b *TaskBus) WaitForTasks() { b.dispatcher.Wait() } // ResizeTaskDispatcher applies a new containers.max_concurrency to the // running global task dispatcher without a restart. Running tasks keep their // slots; a lowered limit is honoured as they finish, and a raised limit // admits queued work immediately. func (b *TaskBus) ResizeTaskDispatcher(maxConcurrency int) { b.dispatcher.SetMaxConcurrency(maxConcurrency) } // submitNATSTask decodes a fetched message and queues it for processing. A // message that cannot be decoded is acked rather than redelivered: it will // never decode on a retry, so leaving it unacked would block the queue. func (b *TaskBus) submitNATSTask(ctx context.Context, msg eventbus.Message) { envelope, err := workintake.DecodeTask(msg.Data()) if err != nil { b.log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err) if err := msg.Ack(); err != nil { b.log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err) } return } task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity} b.dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) { b.processNATSTask(ctx, msg, hold) }) } // processNATSTask decodes a NATS message, writes it to SQLite, claims it, // and runs the workflow. The message is ack'd on terminal (park is a valid // outcome). Nak on transient errors so NATS redelivers. func (b *TaskBus) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) { ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header()) defer span.End() tm, err := workintake.DecodeTask(msg.Data()) if err != nil { b.log.ErrorContext(ctx, "nats decode failed", "err", err) if err := msg.Ack(); err != nil { b.log.WarnContext(ctx, "ack failed", "err", err) } // bad message, don't retry return } if !b.acceptTaskPublisher(ctx, msg, tm.Identity) { return } inserted, err := b.store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity) if err != nil { b.log.ErrorContext(ctx, "nats enqueue failed", "err", err) if err := msg.Nak(); err != nil { b.log.WarnContext(ctx, "nats nak failed", "err", err) } return } if !inserted { // Already tracked -- dedup. Ack and move on. if err := msg.Ack(); err != nil { b.log.WarnContext(ctx, "ack failed", "err", err) } return } task, err := b.store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number) if err != nil { b.log.ErrorContext(ctx, "nats claim failed", "err", err) if err := msg.Nak(); err != nil { b.log.WarnContext(ctx, "nats nak failed", "err", err) } return } if task == nil { // Claimed by another consumer, or task is not queued. Ack. if err := msg.Ack(); err != nil { b.log.WarnContext(ctx, "ack failed", "err", err) } return } b.emitTaskQueued(ctx, task) b.process(ctx, task, hold) if err := msg.Ack(); err != nil { b.log.WarnContext(ctx, "ack failed", "err", err) } } func (b *TaskBus) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool { if ok, reason, class := b.orgs.identityMayAct(ctx, name); !ok { b.log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject()) if class == taskstate.ParkTransient { if err := msg.Nak(); err != nil { b.log.WarnContext(ctx, "nak failed", "err", err) } return false } if err := msg.Ack(); err != nil { b.log.WarnContext(ctx, "ack failed", "err", err) } return false } return true } // emitTaskQueued records the queued task under the identity that polled it, // which is both the actor and the authority the task was created with. func (b *TaskBus) emitTaskQueued(ctx context.Context, task *workflow.Task) { e := events.Event{ Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo, Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt, ActorID: task.Identity, PrincipalID: task.Identity, } if value, err := b.identities.Lookup(ctx, identity.IdentityID(task.Identity)); err == nil { e.ActorKind = string(value.Kind) } b.emit(ctx, e) } // PublishTask validates a task envelope, stamps the producing identity's // org and publishes it. The idempotency key dedups rediscovered issues. func (b *TaskBus) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error { if task.Identity == "" { return fmt.Errorf("publish task %s: no identity", task.Ref()) } resolved, err := b.orgs.identityOrg(ctx, identity.IdentityID(task.Identity)) if err != nil { return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err) } task.Org = resolved payload, err := task.Encode() if err != nil { return err } return b.transport.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload) } // PublishUnique publishes a raw message on the bus, deduplicated by key. func (b *TaskBus) PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error { return b.transport.PublishUnique(ctx, subject, idempotencyKey, payload) } // Request sends a request on the bus and waits for its reply. func (b *TaskBus) Request(ctx context.Context, subject string, payload []byte) ([]byte, error) { return b.transport.Request(ctx, subject, payload) } // InFlight counts the same submitted work that graceful shutdown waits for, // including tasks waiting for a repository or concurrency slot. func (b *TaskBus) InFlight() int { return b.dispatcher.inFlight() } // taskDispatcher limits concurrent tasks globally and runs one task per repo // at a time, in submit order, unless the repo allows concurrency. type taskDispatcher struct { allowConcurrent func(task *workflow.Task) bool mu sync.Mutex limit int active int slotsFree *sync.Cond repoTail map[string]chan struct{} pending int // waiting counts submitted tasks that hold no slot yet, so a drain pass // stops claiming once queued work alone fills the limit. waiting int // holds maps a task id to the capacity its run holds, so a run waiting on // a callee can give that capacity back and take it again by task id. holds map[int64]*taskHold } // SetMaxConcurrency accepts a new global limit without a restart. Running // tasks keep their slots; a lowered limit is enforced as they finish, and a // raised limit admits queued work immediately. func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) { d.mu.Lock() d.limit = maxConcurrency d.mu.Unlock() d.slotsFree.Broadcast() } func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher { if allowConcurrent == nil { allowConcurrent = func(*workflow.Task) bool { return false } } d := &taskDispatcher{ limit: maxConcurrency, allowConcurrent: allowConcurrent, repoTail: make(map[string]chan struct{}), holds: make(map[int64]*taskHold), } d.slotsFree = sync.NewCond(&d.mu) return d } // joinRepoLocked appends baton to the tail of repo's chain and returns the // baton ahead of it, which the joining task is blocked on. Callers hold d.mu. func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} { previous := d.repoTail[repo] d.repoTail[repo] = baton return previous } // takeSlotLocked blocks until the global limit has room, then counts the run // in. limit <= 0 means unlimited. The limit is re-read on every wake, so // SetMaxConcurrency reaches waiting tasks without recreating the dispatcher. // It returns ctx's error, taking nothing, once ctx is done. Callers hold d.mu. func (d *taskDispatcher) takeSlotLocked(ctx context.Context) error { // Broadcasting under the lock keeps the wake from slipping between the ctx // check below and the Wait. stop := context.AfterFunc(ctx, func() { d.mu.Lock() defer d.mu.Unlock() d.slotsFree.Broadcast() }) defer stop() for d.limit > 0 && d.active >= d.limit { if err := ctx.Err(); err != nil { return err } d.slotsFree.Wait() } if err := ctx.Err(); err != nil { return err } d.active++ return nil } // giveBackLocked gives back everything the run holds: its slot under the // global limit, and its place in its repo's chain, which releases the task // queued behind it. Callers hold d.mu. func (d *taskDispatcher) giveBackLocked(hold *taskHold) { if hold.slot { d.active-- hold.slot = false } if hold.baton == nil { return } close(hold.baton) if d.repoTail[hold.repo] == hold.baton { delete(d.repoTail, hold.repo) } hold.baton = nil } // finishRun retires a returned run: it holds no capacity any more, and no // Resume may take any for it. Whatever an in-flight resume takes later is // given back by reacquire, which sees finished. func (d *taskDispatcher) finishRun(hold *taskHold) { d.mu.Lock() hold.finished = true if hold.id != 0 && d.holds[hold.id] == hold { delete(d.holds, hold.id) } d.giveBackLocked(hold) d.pending-- d.mu.Unlock() d.slotsFree.Broadcast() } // setCallWaits applies a durable snapshot without accumulating duplicate events. func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits int) { d.mu.Lock() if d.holds[hold.id] != hold || hold.finished { d.mu.Unlock() return } previous := hold.waits hold.waits = waits if waits > 0 { d.giveBackLocked(hold) } d.mu.Unlock() if waits > 0 { d.slotsFree.Broadcast() } else if previous > 0 { // Reacquiring may block behind another task; the caller keeps running. go d.reacquire(ctx, hold) } } // reacquire returns a resumed run's capacity: its place in its repo's chain // and a slot under the global limit. The run keeps executing meanwhile -- a // wait on a callee is not a pause. A hold records one slot and one place, so a // re-acquire queued while another is still waiting gives back whichever of the // two it finds already recorded instead of taking a second. Once ctx is done, // or the run is stopped, it stops waiting and gives back whatever the hold owns. func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) { d.mu.Lock() if hold.finished || hold.waits > 0 { // The run returned, or started waiting again, after the reconciliation that // queued this: it needs no capacity. d.mu.Unlock() return } stopped := hold.stopped ctx, cancel := context.WithCancel(ctx) defer cancel() if stopped != nil { go func() { select { case <-stopped: cancel() case <-ctx.Done(): } }() } var previous chan struct{} // A place another re-acquire already queued is the run's one place; joining // behind it would queue a second, which the task behind it would wait on // forever. if hold.chain && hold.baton == nil { hold.baton = make(chan struct{}) previous = d.joinRepoLocked(hold.repo, hold.baton) } d.mu.Unlock() if previous != nil { select { case <-previous: case <-ctx.Done(): d.mu.Lock() d.giveBackLocked(hold) d.mu.Unlock() d.slotsFree.Broadcast() return } } d.mu.Lock() if err := d.takeSlotLocked(ctx); err != nil { d.giveBackLocked(hold) d.mu.Unlock() d.slotsFree.Broadcast() return } if hold.slot { // Another re-acquire took the run's one slot while this waited: give // back the slot this one just took. The place is the other's, so it is // left alone. d.active-- d.mu.Unlock() d.slotsFree.Broadcast() return } hold.slot = true if hold.finished || hold.waits > 0 { // The run ended, or started waiting again, while this was queued for // capacity: give back what it took, so no slot or place in the repo // chain leaks. d.giveBackLocked(hold) d.mu.Unlock() d.slotsFree.Broadcast() return } d.mu.Unlock() } // Submit queues a task's process behind the global limit and, unless the repo // allows concurrency, behind the task already holding that repo. process is // handed the run's capacity so it can name it once it knows the task's id and // give it back while it waits on a callee. A run whose ctx ends while it waits // for its repo's place returns without running process. func (d *taskDispatcher) Submit( ctx context.Context, task *workflow.Task, process func(context.Context, *workflow.Task, *taskHold), ) { repo := task.Owner + "/" + task.Repo hold := &taskHold{dispatcher: d, repo: repo} var previous chan struct{} if !d.allowConcurrent(task) { hold.chain = true hold.baton = make(chan struct{}) d.mu.Lock() previous = d.joinRepoLocked(repo, hold.baton) d.mu.Unlock() } d.mu.Lock() d.pending++ d.waiting++ d.mu.Unlock() go func() { defer d.finishRun(hold) if previous != nil { select { case <-previous: case <-ctx.Done(): d.leaveQueue() return } } d.mu.Lock() if err := d.takeSlotLocked(ctx); err != nil { d.mu.Unlock() d.leaveQueue() return } d.waiting-- hold.slot = true d.mu.Unlock() process(ctx, task, hold) }() } // leaveQueue drops a submitted run that was cancelled before it took a slot. func (d *taskDispatcher) leaveQueue() { d.mu.Lock() d.waiting-- d.mu.Unlock() } // hasRoom reports whether a newly submitted task could take a slot without // queueing behind work already submitted. A run waiting on a callee holds no // slot, so its callee still finds room. func (d *taskDispatcher) hasRoom() bool { d.mu.Lock() defer d.mu.Unlock() return d.limit <= 0 || d.active+d.waiting < d.limit } // inFlight is how many submitted tasks have not returned yet. func (d *taskDispatcher) inFlight() int { d.mu.Lock() defer d.mu.Unlock() return d.pending } func (d *taskDispatcher) Wait() { d.mu.Lock() defer d.mu.Unlock() for d.pending > 0 { d.slotsFree.Wait() } }