Files

602 lines
19 KiB
Go

package daemon
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/eventbus"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/natsrpc"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// TaskBus takes work from the NATS stream and the store's requeue fallback,
// runs it under one global dispatcher, and publishes discovered work.
type TaskBus struct {
transport TaskTransport
store storecontract.TaskStore
identities identityLookup
log *slog.Logger
orgs orgResolver
process func(context.Context, *workflow.Task, *taskHold)
emit func(context.Context, events.Event)
markPass func(loop string)
dispatcher *taskDispatcher
}
// TaskBusDeps are the inputs a TaskBus runs on. Every field is required.
type TaskBusDeps struct {
Transport TaskTransport
Store storecontract.TaskStore
Identities identityLookup
Log *slog.Logger
Orgs orgResolver
MaxConcurrency int
AllowConcurrent func(*workflow.Task) bool
Process func(context.Context, *workflow.Task, *taskHold)
Emit func(context.Context, events.Event)
MarkPass func(loop string)
}
// NewTaskBus builds a bus whose dispatcher starts at MaxConcurrency.
func NewTaskBus(deps TaskBusDeps) *TaskBus {
return &TaskBus{
transport: deps.Transport,
store: deps.Store,
identities: deps.Identities,
log: deps.Log,
orgs: deps.Orgs,
process: deps.Process,
emit: deps.Emit,
markPass: deps.MarkPass,
dispatcher: newTaskDispatcher(deps.MaxConcurrency, deps.AllowConcurrent),
}
}
// WireTaskBus builds the daemon's task bus from its fields. The composition
// root calls it once the fields are set, and before the daemon is published:
// process is unexported, so only this package can name it.
func (d *Daemon) WireTaskBus() {
d.TaskBus = NewTaskBus(TaskBusDeps{
Transport: d.TaskTransport,
Store: d.Store,
Identities: d.Identity,
Log: d.Log,
Orgs: d.Identity,
MaxConcurrency: d.Cfg.Get().Containers.MaxConcurrency,
AllowConcurrent: d.allowConcurrentForTask,
Process: d.process,
Emit: d.emit,
MarkPass: d.markPass,
})
}
// identityLookup reads one identity record.
type identityLookup interface {
Lookup(ctx context.Context, id identity.IdentityID) (identity.Identity, error)
}
// drainLoop is the only task drainer and capture dispatcher while Run is
// active. It runs apart from the poll tick so a captured event becomes a
// running task within about a second regardless of poll_interval or forge
// latency. Submit never blocks, so a full dispatcher
// just leaves tasks waiting on slots and the loop stays paced by the ticker.
func (b *TaskBus) drainLoop(ctx context.Context, dispatchBindings func(context.Context)) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
b.markPass("drain")
dispatchBindings(ctx)
b.drainNATS(ctx)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for
// requeued tasks (waiting_approval approval, retry-parked) that didn't come
// through a NATS publish. It returns once the stream and the claim fallback
// are both empty: in-flight tasks keep running, so a task waiting on a callee
// does not hold up the pass that would claim it. Shutdown waits for them
// through WaitForTasks.
func (b *TaskBus) drainNATS(ctx context.Context) {
// A full dispatcher leaves work in the stream and the store, not parked
// on slots where it already reads as running.
for ctx.Err() == nil && b.dispatcher.hasRoom() {
msg, err := b.transport.Fetch(ctx)
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
b.log.ErrorContext(ctx, "nats fetch failed", "err", err)
break
}
if err == nil {
b.submitNATSTask(ctx, msg)
b.markPass("drain")
continue
}
// ErrNoMessage means the stream is drained. That is precisely when
// the SQLite fallback must run: requeued tasks (waiting_approval
// approval, retry-parked) never went through a NATS publish, so
// nothing will ever arrive on the stream to trigger them.
b.claimRequeued(ctx)
return
}
}
// claimRequeued submits tasks from the store's claim fallback while the
// dispatcher has room; the rest stay unclaimed for a later pass. It does
// not go back to the stream between claims: each empty fetch waits a full poll
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
func (b *TaskBus) claimRequeued(ctx context.Context) {
for ctx.Err() == nil && b.dispatcher.hasRoom() {
task, err := b.store.ClaimNext(ctx)
if err != nil {
b.log.ErrorContext(ctx, "sqlite claim failed", "err", err)
return
}
if task == nil {
return
}
b.dispatcher.Submit(ctx, task, b.process)
b.markPass("drain")
}
}
// WaitForTasks blocks until every submitted task's process has returned. A
// drain pass no longer waits for its own tasks, so shutdown is where in-flight
// work is waited for.
func (b *TaskBus) WaitForTasks() {
b.dispatcher.Wait()
}
// ResizeTaskDispatcher applies a new containers.max_concurrency to the
// running global task dispatcher without a restart. Running tasks keep their
// slots; a lowered limit is honoured as they finish, and a raised limit
// admits queued work immediately.
func (b *TaskBus) ResizeTaskDispatcher(maxConcurrency int) {
b.dispatcher.SetMaxConcurrency(maxConcurrency)
}
// submitNATSTask decodes a fetched message and queues it for processing. A
// message that cannot be decoded is acked rather than redelivered: it will
// never decode on a retry, so leaving it unacked would block the queue.
func (b *TaskBus) submitNATSTask(ctx context.Context, msg eventbus.Message) {
envelope, err := workintake.DecodeTask(msg.Data())
if err != nil {
b.log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err)
if err := msg.Ack(); err != nil {
b.log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err)
}
return
}
task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
b.dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) {
b.processNATSTask(ctx, msg, hold)
})
}
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
// and runs the workflow. The message is ack'd on terminal (park is a valid
// outcome). Nak on transient errors so NATS redelivers.
func (b *TaskBus) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) {
ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header())
defer span.End()
tm, err := workintake.DecodeTask(msg.Data())
if err != nil {
b.log.ErrorContext(ctx, "nats decode failed", "err", err)
if err := msg.Ack(); err != nil {
b.log.WarnContext(ctx, "ack failed", "err", err)
}
// bad message, don't retry
return
}
if !b.acceptTaskPublisher(ctx, msg, tm.Identity) {
return
}
inserted, err := b.store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
if err != nil {
b.log.ErrorContext(ctx, "nats enqueue failed", "err", err)
if err := msg.Nak(); err != nil {
b.log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if !inserted {
// Already tracked -- dedup. Ack and move on.
if err := msg.Ack(); err != nil {
b.log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
task, err := b.store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
if err != nil {
b.log.ErrorContext(ctx, "nats claim failed", "err", err)
if err := msg.Nak(); err != nil {
b.log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if task == nil {
// Claimed by another consumer, or task is not queued. Ack.
if err := msg.Ack(); err != nil {
b.log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
b.emitTaskQueued(ctx, task)
b.process(ctx, task, hold)
if err := msg.Ack(); err != nil {
b.log.WarnContext(ctx, "ack failed", "err", err)
}
}
func (b *TaskBus) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool {
if ok, reason, class := b.orgs.identityMayAct(ctx, name); !ok {
b.log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject())
if class == taskstate.ParkTransient {
if err := msg.Nak(); err != nil {
b.log.WarnContext(ctx, "nak failed", "err", err)
}
return false
}
if err := msg.Ack(); err != nil {
b.log.WarnContext(ctx, "ack failed", "err", err)
}
return false
}
return true
}
// emitTaskQueued records the queued task under the identity that polled it,
// which is both the actor and the authority the task was created with.
func (b *TaskBus) emitTaskQueued(ctx context.Context, task *workflow.Task) {
e := events.Event{
Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo,
Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt,
ActorID: task.Identity, PrincipalID: task.Identity,
}
if value, err := b.identities.Lookup(ctx, identity.IdentityID(task.Identity)); err == nil {
e.ActorKind = string(value.Kind)
}
b.emit(ctx, e)
}
// PublishTask validates a task envelope, stamps the producing identity's
// org and publishes it. The idempotency key dedups rediscovered issues.
func (b *TaskBus) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
if task.Identity == "" {
return fmt.Errorf("publish task %s: no identity", task.Ref())
}
resolved, err := b.orgs.identityOrg(ctx, identity.IdentityID(task.Identity))
if err != nil {
return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err)
}
task.Org = resolved
payload, err := task.Encode()
if err != nil {
return err
}
return b.transport.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
}
// PublishUnique publishes a raw message on the bus, deduplicated by key.
func (b *TaskBus) PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error {
return b.transport.PublishUnique(ctx, subject, idempotencyKey, payload)
}
// Request sends a request on the bus and waits for its reply.
func (b *TaskBus) Request(ctx context.Context, subject string, payload []byte) ([]byte, error) {
return b.transport.Request(ctx, subject, payload)
}
// InFlight counts the same submitted work that graceful shutdown waits for,
// including tasks waiting for a repository or concurrency slot.
func (b *TaskBus) InFlight() int {
return b.dispatcher.inFlight()
}
// taskDispatcher limits concurrent tasks globally and runs one task per repo
// at a time, in submit order, unless the repo allows concurrency.
type taskDispatcher struct {
allowConcurrent func(task *workflow.Task) bool
mu sync.Mutex
limit int
active int
slotsFree *sync.Cond
repoTail map[string]chan struct{}
pending int
// waiting counts submitted tasks that hold no slot yet, so a drain pass
// stops claiming once queued work alone fills the limit.
waiting int
// holds maps a task id to the capacity its run holds, so a run waiting on
// a callee can give that capacity back and take it again by task id.
holds map[int64]*taskHold
}
// SetMaxConcurrency accepts a new global limit without a restart. Running
// tasks keep their slots; a lowered limit is enforced as they finish, and a
// raised limit admits queued work immediately.
func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) {
d.mu.Lock()
d.limit = maxConcurrency
d.mu.Unlock()
d.slotsFree.Broadcast()
}
func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher {
if allowConcurrent == nil {
allowConcurrent = func(*workflow.Task) bool { return false }
}
d := &taskDispatcher{
limit: maxConcurrency,
allowConcurrent: allowConcurrent,
repoTail: make(map[string]chan struct{}),
holds: make(map[int64]*taskHold),
}
d.slotsFree = sync.NewCond(&d.mu)
return d
}
// joinRepoLocked appends baton to the tail of repo's chain and returns the
// baton ahead of it, which the joining task is blocked on. Callers hold d.mu.
func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} {
previous := d.repoTail[repo]
d.repoTail[repo] = baton
return previous
}
// takeSlotLocked blocks until the global limit has room, then counts the run
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
// It returns ctx's error, taking nothing, once ctx is done. Callers hold d.mu.
func (d *taskDispatcher) takeSlotLocked(ctx context.Context) error {
// Broadcasting under the lock keeps the wake from slipping between the ctx
// check below and the Wait.
stop := context.AfterFunc(ctx, func() {
d.mu.Lock()
defer d.mu.Unlock()
d.slotsFree.Broadcast()
})
defer stop()
for d.limit > 0 && d.active >= d.limit {
if err := ctx.Err(); err != nil {
return err
}
d.slotsFree.Wait()
}
if err := ctx.Err(); err != nil {
return err
}
d.active++
return nil
}
// giveBackLocked gives back everything the run holds: its slot under the
// global limit, and its place in its repo's chain, which releases the task
// queued behind it. Callers hold d.mu.
func (d *taskDispatcher) giveBackLocked(hold *taskHold) {
if hold.slot {
d.active--
hold.slot = false
}
if hold.baton == nil {
return
}
close(hold.baton)
if d.repoTail[hold.repo] == hold.baton {
delete(d.repoTail, hold.repo)
}
hold.baton = nil
}
// finishRun retires a returned run: it holds no capacity any more, and no
// Resume may take any for it. Whatever an in-flight resume takes later is
// given back by reacquire, which sees finished.
func (d *taskDispatcher) finishRun(hold *taskHold) {
d.mu.Lock()
hold.finished = true
if hold.id != 0 && d.holds[hold.id] == hold {
delete(d.holds, hold.id)
}
d.giveBackLocked(hold)
d.pending--
d.mu.Unlock()
d.slotsFree.Broadcast()
}
// setCallWaits applies a durable snapshot without accumulating duplicate events.
func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits int) {
d.mu.Lock()
if d.holds[hold.id] != hold || hold.finished {
d.mu.Unlock()
return
}
previous := hold.waits
hold.waits = waits
if waits > 0 {
d.giveBackLocked(hold)
}
d.mu.Unlock()
if waits > 0 {
d.slotsFree.Broadcast()
} else if previous > 0 {
// Reacquiring may block behind another task; the caller keeps running.
go d.reacquire(ctx, hold)
}
}
// reacquire returns a resumed run's capacity: its place in its repo's chain
// and a slot under the global limit. The run keeps executing meanwhile -- a
// wait on a callee is not a pause. A hold records one slot and one place, so a
// re-acquire queued while another is still waiting gives back whichever of the
// two it finds already recorded instead of taking a second. Once ctx is done,
// or the run is stopped, it stops waiting and gives back whatever the hold owns.
func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
d.mu.Lock()
if hold.finished || hold.waits > 0 {
// The run returned, or started waiting again, after the reconciliation that
// queued this: it needs no capacity.
d.mu.Unlock()
return
}
stopped := hold.stopped
ctx, cancel := context.WithCancel(ctx)
defer cancel()
if stopped != nil {
go func() {
select {
case <-stopped:
cancel()
case <-ctx.Done():
}
}()
}
var previous chan struct{}
// A place another re-acquire already queued is the run's one place; joining
// behind it would queue a second, which the task behind it would wait on
// forever.
if hold.chain && hold.baton == nil {
hold.baton = make(chan struct{})
previous = d.joinRepoLocked(hold.repo, hold.baton)
}
d.mu.Unlock()
if previous != nil {
select {
case <-previous:
case <-ctx.Done():
d.mu.Lock()
d.giveBackLocked(hold)
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
}
d.mu.Lock()
if err := d.takeSlotLocked(ctx); err != nil {
d.giveBackLocked(hold)
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
if hold.slot {
// Another re-acquire took the run's one slot while this waited: give
// back the slot this one just took. The place is the other's, so it is
// left alone.
d.active--
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
hold.slot = true
if hold.finished || hold.waits > 0 {
// The run ended, or started waiting again, while this was queued for
// capacity: give back what it took, so no slot or place in the repo
// chain leaks.
d.giveBackLocked(hold)
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
d.mu.Unlock()
}
// Submit queues a task's process behind the global limit and, unless the repo
// allows concurrency, behind the task already holding that repo. process is
// handed the run's capacity so it can name it once it knows the task's id and
// give it back while it waits on a callee. A run whose ctx ends while it waits
// for its repo's place returns without running process.
func (d *taskDispatcher) Submit(
ctx context.Context,
task *workflow.Task,
process func(context.Context, *workflow.Task, *taskHold),
) {
repo := task.Owner + "/" + task.Repo
hold := &taskHold{dispatcher: d, repo: repo}
var previous chan struct{}
if !d.allowConcurrent(task) {
hold.chain = true
hold.baton = make(chan struct{})
d.mu.Lock()
previous = d.joinRepoLocked(repo, hold.baton)
d.mu.Unlock()
}
d.mu.Lock()
d.pending++
d.waiting++
d.mu.Unlock()
go func() {
defer d.finishRun(hold)
if previous != nil {
select {
case <-previous:
case <-ctx.Done():
d.leaveQueue()
return
}
}
d.mu.Lock()
if err := d.takeSlotLocked(ctx); err != nil {
d.mu.Unlock()
d.leaveQueue()
return
}
d.waiting--
hold.slot = true
d.mu.Unlock()
process(ctx, task, hold)
}()
}
// leaveQueue drops a submitted run that was cancelled before it took a slot.
func (d *taskDispatcher) leaveQueue() {
d.mu.Lock()
d.waiting--
d.mu.Unlock()
}
// hasRoom reports whether a newly submitted task could take a slot without
// queueing behind work already submitted. A run waiting on a callee holds no
// slot, so its callee still finds room.
func (d *taskDispatcher) hasRoom() bool {
d.mu.Lock()
defer d.mu.Unlock()
return d.limit <= 0 || d.active+d.waiting < d.limit
}
// inFlight is how many submitted tasks have not returned yet.
func (d *taskDispatcher) inFlight() int {
d.mu.Lock()
defer d.mu.Unlock()
return d.pending
}
func (d *taskDispatcher) Wait() {
d.mu.Lock()
defer d.mu.Unlock()
for d.pending > 0 {
d.slotsFree.Wait()
}
}