fix(daemon): the task bus is its own type; a cancelled run stops waiting for its slot (#1932)

This commit is contained in:
sam committed 2026-11-13 04:17:46 +11:00
1 parent 0cd9cefd76
commit 500d02be78
15 files changed
+273 -150

No files matched your search

+9 -7
View File
@@ -592,7 +592,7 @@ func (b *boot) issueForge(ctx context.Context, owner, repo string) (forge.IssueF
func (b *boot) buildDaemon(ctx context.Context) {
log := b.log
b.d = &daemon.Daemon{
d := &daemon.Daemon{
ResolveRepository: b.repositoryAccess,
Cfg: b.cfgHolder,
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal},
@@ -609,7 +609,7 @@ func (b *boot) buildDaemon(ctx context.Context) {
PackageAuthorities: installedAuthorities{b.stateStoreGrants.Client},
Storage: b.storeBackend,
Log: log,
Tasks: b.natsClient,
TaskTransport: b.natsClient,
RunCredentials: b.stateStoreGrants,
ContainerPool: b.containerPool,
ModelEgress: b.modelEgress,
@@ -621,9 +621,9 @@ func (b *boot) buildDaemon(ctx context.Context) {
Playbooks: b.livePlaybooks(),
}
b.d.Access = b.accessChain
b.d.ReviewScanner = daemon.NewReviewScanner(b.stateStore, b.reactionClient, b.natsClient.PublishUnique, b.bus, b.d, b.d, b.d, log)
b.d.BindingDispatch = daemon.NewBindingDispatch(daemon.BindingDispatchDeps{
d.Access = b.accessChain
d.ReviewScanner = daemon.NewReviewScanner(b.stateStore, b.reactionClient, b.natsClient.PublishUnique, b.bus, d, d, d, log)
d.BindingDispatch = daemon.NewBindingDispatch(daemon.BindingDispatchDeps{
Store: b.stateStore,
Mappings: b.stateClient,
Bindings: b.stateClient,
@@ -635,11 +635,13 @@ func (b *boot) buildDaemon(ctx context.Context) {
Access: b.accessChain,
Denials: b.stateClient,
Cfg: b.cfgHolder,
Orgs: b.d,
Orgs: d,
OrgPrincipals: b.orgPrincipals,
Log: log,
})
b.d.IssuePoller = daemon.NewIssuePoller(b.cfgHolder, b.stateStore, log, b.issueSources, b.taskActions().Withdraw, b.d.PublishTask, b.d, b.d.BindingDispatch)
d.WireTaskBus()
d.IssuePoller = daemon.NewIssuePoller(b.cfgHolder, b.stateStore, log, b.issueSources, b.taskActions().Withdraw, d.TaskBus.PublishTask, d, d.BindingDispatch)
b.d = d
}
func (b *boot) publishConfig(ctx context.Context, cfg config.Config) {
+1 -1
View File
@@ -405,7 +405,7 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda
// dispatcher is created later from the config the next drain reads.
func (b *boot) resizeTaskDispatcher(maxConcurrency int) {
if b.d != nil {
b.d.ResizeTaskDispatcher(maxConcurrency)
b.d.TaskBus.ResizeTaskDispatcher(maxConcurrency)
}
}
+1 -1
View File
@@ -94,7 +94,7 @@ func (b *boot) startHealth(ctx context.Context) error {
if b.d == nil {
return -1
}
return b.d.InFlight()
return b.d.TaskBus.InFlight()
},
}
addr := b.cfg.Health.Listen
+4 -4
View File
@@ -65,10 +65,10 @@ func TestRunViaAgentRequiresTerminalStatus(t *testing.T) {
}
store := &statusStore{}
d := wired(&Daemon{
Cfg: config.NewHolder(config.Config{}),
Store: store,
Tasks: statusBus{reply: reply},
Log: slog.New(slog.DiscardHandler),
Cfg: config.NewHolder(config.Config{}),
Store: store,
TaskTransport: statusBus{reply: reply},
Log: slog.New(slog.DiscardHandler),
})
d.runViaAgent(t.Context(), &workflow.Task{ID: 7, Attempt: 1, Owner: "owner", Repo: "repo", Workflow: "implement"},
config.Repo{}, config.AgentProfile{}, "")
+10 -10
View File
@@ -25,7 +25,7 @@ func (d *Daemon) watchCallWaits(ctx context.Context) {
return
}
sub := d.Bus.Subscribe(callWaitBuffer)
dispatcher := d.taskDispatcher()
bus := d.TaskBus
go func() {
defer sub.Close()
ticker := time.NewTicker(time.Second)
@@ -39,29 +39,29 @@ func (d *Daemon) watchCallWaits(ctx context.Context) {
return
}
if e.Kind == events.KindWorkflowCallStarted || e.Kind == events.KindWorkflowCallFinished || e.Kind == events.KindStageStart || e.Kind == events.KindStageFinish {
d.reconcileCallWaits(ctx, dispatcher, reader)
bus.reconcileCallWaits(ctx, reader)
}
case <-ticker.C:
d.reconcileCallWaits(ctx, dispatcher, reader)
bus.reconcileCallWaits(ctx, reader)
}
}
}()
}
// Events only prompt a read; periodic reads recover dropped starts and finishes.
func (d *Daemon) reconcileCallWaits(ctx context.Context, dispatcher *taskDispatcher, reader storecontract.StepReader) {
dispatcher.mu.Lock()
holds := make([]*taskHold, 0, len(dispatcher.holds))
for _, hold := range dispatcher.holds {
func (b *TaskBus) reconcileCallWaits(ctx context.Context, reader storecontract.StepReader) {
b.dispatcher.mu.Lock()
holds := make([]*taskHold, 0, len(b.dispatcher.holds))
for _, hold := range b.dispatcher.holds {
holds = append(holds, hold)
}
dispatcher.mu.Unlock()
b.dispatcher.mu.Unlock()
for _, hold := range holds {
readCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
steps, err := reader.ListSteps(readCtx, hold.id, hold.attempt)
cancel()
if err != nil {
d.Log.WarnContext(ctx, "read call waits", "task", hold.id, "err", err)
b.log.WarnContext(ctx, "read call waits", "task", hold.id, "err", err)
continue
}
waits := 0
@@ -70,6 +70,6 @@ func (d *Daemon) reconcileCallWaits(ctx context.Context, dispatcher *taskDispatc
waits++
}
}
dispatcher.setCallWaits(hold, waits)
b.dispatcher.setCallWaits(ctx, hold, waits)
}
}
+1 -1
View File
@@ -306,7 +306,7 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte)
deadline := time.Now().Add(taskRunReadyTimeout)
backoff := taskRunRetryBackoff
for {
reply, err := d.Tasks.Request(ctx, subject, data)
reply, err := d.TaskBus.Request(ctx, subject, data)
if err == nil {
return reply, nil
}
+8 -11
View File
@@ -29,9 +29,9 @@ import (
"github.com/samcharles93/archie-core/internal/worktree"
)
// TaskBus publishes discovered work, collects it back, and asks a worker to
// run a task.
type TaskBus interface {
// TaskTransport publishes discovered work, collects it back, and asks a worker
// to run a task. The daemon's TaskBus runs on it.
type TaskTransport interface {
PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error
Fetch(ctx context.Context) (eventbus.Message, error)
Request(ctx context.Context, subject string, payload []byte) ([]byte, error)
@@ -100,8 +100,11 @@ type Daemon struct {
Trees *worktree.Manager
Bus *events.Bus
Log *slog.Logger
// Tasks is the NATS task bus.
Tasks TaskBus
// TaskTransport is the NATS connection the task bus runs on. The bus is
// built by WireTaskBus.
TaskTransport TaskTransport
// TaskBus takes, runs and publishes tasks. Set by WireTaskBus.
TaskBus *TaskBus
// ReviewScanner reconciles PRs, scans their reviews and drains review
// reactions into remediate runs.
ReviewScanner *ReviewScanner
@@ -125,12 +128,6 @@ type Daemon struct {
// execution is unavailable and process parks tasks; every runnable task gets
// a fresh container.
ContainerPool *container.Pool
// dispatcher bounds task execution globally across poll passes. Built once,
// on first use, so ResizeTaskDispatcher can apply a live
// containers.max_concurrency change without a restart; drainNATS submits
// each pass through it and leaves its tasks running.
dispatcherOnce sync.Once
dispatcher *taskDispatcher
// Storage is the pluggable storage backend for container mounts.
// A runnable task requires it; acquireTaskContainer parks on nil.
Storage storage.Backend
+59 -20
View File
@@ -150,7 +150,7 @@ func changeCallWaits(d *taskDispatcher, delta int, id int64) {
}
d.mu.Unlock()
if hold != nil {
d.setCallWaits(hold, waits)
d.setCallWaits(context.Background(), hold, waits)
}
}
@@ -339,7 +339,7 @@ func TestSecondReacquireDoesNotQueueASecondPlace(t *testing.T) {
// A second re-acquire for the same run, with the first still waiting. A
// Resume cannot queue this one: queueing two takes a Suspend between them,
// and that Suspend would clear the place queued above.
go d.reacquire(taskHoldOf(d, 1))
go d.reacquire(context.Background(), taskHoldOf(d, 1))
awaitNoState(t, "a second place was queued for the run", func() bool { return placeClaim(d, 1) != queued })
awaitState(t, "task 1 did not take its slot back", func() bool { return holdsCapacity(d, 1) })
@@ -436,7 +436,7 @@ func TestCallWaitsRecoverDroppedStart(t *testing.T) {
cfg := config.Config{}
cfg.Containers.MaxConcurrency = 1
d := wired(&Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)})
dispatcher := d.taskDispatcher()
dispatcher := d.TaskBus.dispatcher
started := make(chan int64, 2)
a, b := make(chan struct{}), make(chan struct{})
defer func() { close(a); dispatcher.Wait() }()
@@ -526,15 +526,15 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) {
},
}
d := wired(&Daemon{
Cfg: config.NewHolder(config.Config{}),
Log: slog.New(slog.DiscardHandler),
Store: store,
Tasks: stubTasks{},
Cfg: config.NewHolder(config.Config{}),
Log: slog.New(slog.DiscardHandler),
Store: store,
TaskTransport: stubTasks{},
})
drained := make(chan struct{})
go func() {
d.drainNATS(context.Background())
d.TaskBus.drainNATS(context.Background())
close(drained)
}()
@@ -552,7 +552,7 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) {
// Shutdown is where that wait now lives.
waited := make(chan struct{})
go func() {
d.WaitForTasks()
d.TaskBus.WaitForTasks()
close(waited)
}()
select {
@@ -605,13 +605,13 @@ func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) {
}
tasks := &countingTasks{}
d := wired(&Daemon{
Cfg: config.NewHolder(config.Config{}),
Log: slog.New(slog.DiscardHandler),
Store: store,
Tasks: tasks,
Cfg: config.NewHolder(config.Config{}),
Log: slog.New(slog.DiscardHandler),
Store: store,
TaskTransport: tasks,
})
d.drainNATS(context.Background())
d.WaitForTasks()
d.TaskBus.drainNATS(context.Background())
d.TaskBus.WaitForTasks()
if len(store.queue) != 0 {
t.Fatalf("%d tasks left unclaimed", len(store.queue))
}
@@ -633,13 +633,52 @@ func TestDrainPassClaimsOnlyWhatTheDispatcherCanRun(t *testing.T) {
cfg := config.Config{}
cfg.Containers.MaxConcurrency = 2
d := wired(&Daemon{
Cfg: config.NewHolder(cfg),
Log: slog.New(slog.DiscardHandler),
Store: store,
Tasks: &countingTasks{},
Cfg: config.NewHolder(cfg),
Log: slog.New(slog.DiscardHandler),
Store: store,
TaskTransport: &countingTasks{},
})
d.drainNATS(context.Background())
d.TaskBus.drainNATS(context.Background())
if got := len(store.queue); got != 8 {
t.Fatalf("pass left %d tasks unclaimed, want 8", got)
}
}
// A run queued behind another in its repo's chain returns once its ctx is
// cancelled, so shutdown's wait is not held by a run that never gets a turn.
// Before the fix it waited on the chain forever and left pending and waiting
// counted.
func TestCancelledRunQueuedBehindItsRepoReturns(t *testing.T) {
d := newTaskDispatcher(2, nil)
started := make(chan int64, 4)
a := make(chan struct{})
submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1)
ctx, cancel := context.WithCancel(t.Context())
d.Submit(ctx, &workflow.Task{ID: 2, Owner: "acme", Repo: "app"}, func(context.Context, *workflow.Task, *taskHold) {
started <- 2
})
awaitNoStart(t, started, "while task 1 held the repo's place")
cancel()
awaitState(t, "the cancelled run is still counted as pending", func() bool { return d.inFlight() == 1 })
close(a)
waited := make(chan struct{})
go func() {
d.Wait()
close(waited)
}()
select {
case <-waited:
case <-time.After(runDeadline):
t.Fatal("Wait did not return: the cancelled run never gave up its place")
}
awaitNoStart(t, started, "after its ctx was cancelled")
d.mu.Lock()
defer d.mu.Unlock()
if d.pending != 0 || d.waiting != 0 {
t.Fatalf("pending = %d, waiting = %d after every run returned, want 0 and 0", d.pending, d.waiting)
}
}
+1 -1
View File
@@ -55,7 +55,7 @@ func TestPublishDiscoveredCap(t *testing.T) {
for _, n := range tt.existing {
have[n] = true
}
d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus})
d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, TaskTransport: bus})
var issues []forge.Issue
for n := 1; n <= tt.issues; n++ {
issues = append(issues, forge.Issue{Number: n, Labels: []string{"archie"}})
+1 -1
View File
@@ -91,7 +91,7 @@ func TestPollNATSSkipsDisabledRoutedWorkflow(t *testing.T) {
fg := &reactCounter{}
var logs bytes.Buffer
d := wired(&Daemon{
Log: slog.New(slog.NewTextHandler(&logs, nil)), Store: taskLookup{}, Tasks: bus,
Log: slog.New(slog.NewTextHandler(&logs, nil)), Store: taskLookup{}, TaskTransport: bus,
WorkflowDefinitions: tt.definitions,
Principals: tt.principals,
})
+6 -6
View File
@@ -81,13 +81,13 @@ const sweepTimeout = 2 * time.Minute
func (d *Daemon) Run(ctx context.Context) error {
// A pass no longer waits for the tasks it submitted, so the run does, once
// the ctx is cancelled.
defer d.WaitForTasks()
defer d.TaskBus.WaitForTasks()
d.watchCallWaits(ctx)
drained := make(chan struct{})
defer func() { <-drained }()
go func() {
defer close(drained)
d.drainLoop(ctx)
d.TaskBus.drainLoop(ctx, d.BindingDispatch.dispatchBindings)
}()
interval := d.Cfg.Get().PollInterval.Std()
ticker := time.NewTicker(interval)
@@ -112,7 +112,7 @@ func (d *Daemon) Run(ctx context.Context) error {
// on a callee stays in flight while the callee sits queued, so the queue is
// drained again until every task, callees included, has returned.
func (d *Daemon) RunOnce(ctx context.Context) {
defer d.WaitForTasks()
defer d.TaskBus.WaitForTasks()
d.watchCallWaits(ctx)
// One pass has no scheduler, so it runs the maintenance jobs itself.
for _, job := range d.BuiltinJobs() {
@@ -122,15 +122,15 @@ func (d *Daemon) RunOnce(ctx context.Context) {
}
d.BindingDispatch.dispatchBindings(ctx)
d.Cycle(ctx)
d.drainNATS(ctx)
d.TaskBus.drainNATS(ctx)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for d.taskDispatcher().inFlight() > 0 {
for d.TaskBus.InFlight() > 0 {
select {
case <-ctx.Done():
return
case <-ticker.C:
d.drainNATS(ctx)
d.TaskBus.drainNATS(ctx)
}
}
}
+2 -2
View File
@@ -96,8 +96,8 @@ func TestRunViaAgentCarriesProviderCredentialVariable(t *testing.T) {
"used": {Class: "deepseek", APIKey: config.SecretRef{Engine: "env", Key: "USED_KEY"}},
},
}),
Tasks: bus,
Log: slog.New(slog.DiscardHandler),
TaskTransport: bus,
Log: slog.New(slog.DiscardHandler),
})
task := &workflow.Task{ID: 7, Identity: string(sys), Attempt: 1, Workflow: "implement", WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"}
+1 -1
View File
@@ -55,7 +55,7 @@ func TestPolledTaskKeepsItsCreatingIdentity(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
bus := &publishedKeys{}
d := wired(&Daemon{
Log: slog.New(slog.DiscardHandler), Store: taskLookup{}, Tasks: bus,
Log: slog.New(slog.DiscardHandler), Store: taskLookup{}, TaskTransport: bus,
Cfg: config.NewHolder(config.Config{}),
})
d.IssuePoller.sources = func(context.Context) []IssueSource {
+156 -75
View File
@@ -4,11 +4,13 @@ import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
"time"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/eventbus"
@@ -17,18 +19,79 @@ import (
"github.com/samcharles93/archie-core/internal/taskstate"
)
// TaskBus takes work from the NATS stream and the store's requeue fallback,
// runs it under one global dispatcher, and publishes discovered work.
type TaskBus struct {
transport TaskTransport
store storecontract.TaskStore
identities identity.Repository
log *slog.Logger
orgs orgResolver
process func(context.Context, *workflow.Task, *taskHold)
emit func(context.Context, events.Event)
markPass func(loop string)
dispatcher *taskDispatcher
}
// TaskBusDeps are the inputs a TaskBus runs on. Every field is required.
type TaskBusDeps struct {
Transport TaskTransport
Store storecontract.TaskStore
Identities identity.Repository
Log *slog.Logger
Orgs orgResolver
MaxConcurrency int
AllowConcurrent func(*workflow.Task) bool
Process func(context.Context, *workflow.Task, *taskHold)
Emit func(context.Context, events.Event)
MarkPass func(loop string)
}
// NewTaskBus builds a bus whose dispatcher starts at MaxConcurrency.
func NewTaskBus(deps TaskBusDeps) *TaskBus {
return &TaskBus{
transport: deps.Transport,
store: deps.Store,
identities: deps.Identities,
log: deps.Log,
orgs: deps.Orgs,
process: deps.Process,
emit: deps.Emit,
markPass: deps.MarkPass,
dispatcher: newTaskDispatcher(deps.MaxConcurrency, deps.AllowConcurrent),
}
}
// WireTaskBus builds the daemon's task bus from its fields. The composition
// root calls it once the fields are set, and before the daemon is published:
// process is unexported, so only this package can name it.
func (d *Daemon) WireTaskBus() {
d.TaskBus = NewTaskBus(TaskBusDeps{
Transport: d.TaskTransport,
Store: d.Store,
Identities: d.IdentityRepository,
Log: d.Log,
Orgs: d,
MaxConcurrency: d.Cfg.Get().Containers.MaxConcurrency,
AllowConcurrent: d.allowConcurrentForTask,
Process: d.process,
Emit: d.emit,
MarkPass: d.markPass,
})
}
// drainLoop is the only task drainer and capture dispatcher while Run is
// active. It runs apart from the poll tick so a captured event becomes a
// running task within about a second regardless of poll_interval or forge
// latency. Submit never blocks, so a full dispatcher
// just leaves tasks waiting on slots and the loop stays paced by the ticker.
func (d *Daemon) drainLoop(ctx context.Context) {
func (b *TaskBus) drainLoop(ctx context.Context, dispatchBindings func(context.Context)) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
d.markPass("drain")
d.BindingDispatch.dispatchBindings(ctx)
d.drainNATS(ctx)
b.markPass("drain")
dispatchBindings(ctx)
b.drainNATS(ctx)
select {
case <-ctx.Done():
return
@@ -43,19 +106,18 @@ func (d *Daemon) drainLoop(ctx context.Context) {
// are both empty: in-flight tasks keep running, so a task waiting on a callee
// does not hold up the pass that would claim it. Shutdown waits for them
// through WaitForTasks.
func (d *Daemon) drainNATS(ctx context.Context) {
dispatcher := d.taskDispatcher()
func (b *TaskBus) drainNATS(ctx context.Context) {
// A full dispatcher leaves work in the stream and the store, not parked
// on slots where it already reads as running.
for ctx.Err() == nil && dispatcher.hasRoom() {
msg, err := d.Tasks.Fetch(ctx)
for ctx.Err() == nil && b.dispatcher.hasRoom() {
msg, err := b.transport.Fetch(ctx)
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
d.Log.ErrorContext(ctx, "nats fetch failed", "err", err)
b.log.ErrorContext(ctx, "nats fetch failed", "err", err)
break
}
if err == nil {
d.submitNATSTask(ctx, dispatcher, msg)
d.markPass("drain")
b.submitNATSTask(ctx, msg)
b.markPass("drain")
continue
}
@@ -63,7 +125,7 @@ func (d *Daemon) drainNATS(ctx context.Context) {
// the SQLite fallback must run: requeued tasks (waiting_approval
// approval, retry-parked) never went through a NATS publish, so
// nothing will ever arrive on the stream to trigger them.
d.claimRequeued(ctx, dispatcher)
b.claimRequeued(ctx)
return
}
}
@@ -72,134 +134,124 @@ func (d *Daemon) drainNATS(ctx context.Context) {
// dispatcher has room; the rest stay unclaimed for a later pass. It does
// not go back to the stream between claims: each empty fetch waits a full poll
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
for ctx.Err() == nil && dispatcher.hasRoom() {
task, err := d.Store.ClaimNext(ctx)
func (b *TaskBus) claimRequeued(ctx context.Context) {
for ctx.Err() == nil && b.dispatcher.hasRoom() {
task, err := b.store.ClaimNext(ctx)
if err != nil {
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
b.log.ErrorContext(ctx, "sqlite claim failed", "err", err)
return
}
if task == nil {
return
}
dispatcher.Submit(ctx, task, d.process)
d.markPass("drain")
b.dispatcher.Submit(ctx, task, b.process)
b.markPass("drain")
}
}
// taskDispatcher returns the daemon's long-lived global dispatcher, building
// it from the running config on first use. It outlives a single drain pass so
// ResizeTaskDispatcher can resize the running dispatcher.
func (d *Daemon) taskDispatcher() *taskDispatcher {
d.dispatcherOnce.Do(func() {
d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask)
})
return d.dispatcher
}
// WaitForTasks blocks until every submitted task's process has returned. A
// drain pass no longer waits for its own tasks, so shutdown is where in-flight
// work is waited for.
func (d *Daemon) WaitForTasks() {
d.taskDispatcher().Wait()
func (b *TaskBus) WaitForTasks() {
b.dispatcher.Wait()
}
// ResizeTaskDispatcher applies a new containers.max_concurrency to the
// running global task dispatcher without a restart. Running tasks keep their
// slots; a lowered limit is honoured as they finish, and a raised limit
// admits queued work immediately.
func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) {
d.taskDispatcher().SetMaxConcurrency(maxConcurrency)
func (b *TaskBus) ResizeTaskDispatcher(maxConcurrency int) {
b.dispatcher.SetMaxConcurrency(maxConcurrency)
}
// submitNATSTask decodes a fetched message and queues it for processing. A
// message that cannot be decoded is acked rather than redelivered: it will
// never decode on a retry, so leaving it unacked would block the queue.
func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) {
func (b *TaskBus) submitNATSTask(ctx context.Context, msg eventbus.Message) {
envelope, err := workintake.DecodeTask(msg.Data())
if err != nil {
d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err)
b.log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err)
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err)
b.log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err)
}
return
}
task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) {
d.processNATSTask(ctx, msg, hold)
b.dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) {
b.processNATSTask(ctx, msg, hold)
})
}
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
// and runs the workflow. The message is ack'd on terminal (park is a valid
// outcome). Nak on transient errors so NATS redelivers.
func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) {
func (b *TaskBus) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) {
ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header())
defer span.End()
tm, err := workintake.DecodeTask(msg.Data())
if err != nil {
d.Log.ErrorContext(ctx, "nats decode failed", "err", err)
b.log.ErrorContext(ctx, "nats decode failed", "err", err)
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
b.log.WarnContext(ctx, "ack failed", "err", err)
}
// bad message, don't retry
return
}
if !d.acceptTaskPublisher(ctx, msg, tm.Identity) {
if !b.acceptTaskPublisher(ctx, msg, tm.Identity) {
return
}
inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
inserted, err := b.store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
if err != nil {
d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err)
b.log.ErrorContext(ctx, "nats enqueue failed", "err", err)
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
b.log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if !inserted {
// Already tracked -- dedup. Ack and move on.
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
b.log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
task, err := b.store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
if err != nil {
d.Log.ErrorContext(ctx, "nats claim failed", "err", err)
b.log.ErrorContext(ctx, "nats claim failed", "err", err)
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
b.log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if task == nil {
// Claimed by another consumer, or task is not queued. Ack.
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
b.log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
d.emitTaskQueued(ctx, task)
d.process(ctx, task, hold)
b.emitTaskQueued(ctx, task)
b.process(ctx, task, hold)
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
b.log.WarnContext(ctx, "ack failed", "err", err)
}
}
func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool {
if ok, reason, class := d.identityMayAct(ctx, name); !ok {
d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject())
func (b *TaskBus) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool {
if ok, reason, class := b.orgs.identityMayAct(ctx, name); !ok {
b.log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject())
if class == taskstate.ParkTransient {
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nak failed", "err", err)
b.log.WarnContext(ctx, "nak failed", "err", err)
}
return false
}
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
b.log.WarnContext(ctx, "ack failed", "err", err)
}
return false
}
@@ -209,28 +261,28 @@ func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message,
// emitTaskQueued records the queued task under the identity that polled it,
// which is both the actor and the authority the task was created with.
func (d *Daemon) emitTaskQueued(ctx context.Context, task *workflow.Task) {
func (b *TaskBus) emitTaskQueued(ctx context.Context, task *workflow.Task) {
e := events.Event{
Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo,
Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt,
ActorID: task.Identity, PrincipalID: task.Identity,
}
if value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(task.Identity)); err == nil {
if value, err := b.identities.Get(ctx, identity.IdentityID(task.Identity)); err == nil {
e.ActorKind = string(value.Kind)
}
d.emit(ctx, e)
b.emit(ctx, e)
}
// PublishTask validates a task envelope, stamps the producing identity's
// org and publishes it. The idempotency key dedups rediscovered issues.
func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
func (b *TaskBus) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
if err := task.Kind.Validate(); err != nil {
return fmt.Errorf("publish task %s: %w", task.Ref(), err)
}
if task.Identity == "" {
return fmt.Errorf("publish task %s: no identity", task.Ref())
}
resolved, err := d.identityOrg(ctx, identity.IdentityID(task.Identity))
resolved, err := b.orgs.identityOrg(ctx, identity.IdentityID(task.Identity))
if err != nil {
return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err)
}
@@ -239,16 +291,23 @@ func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope)
if err != nil {
return err
}
return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
return b.transport.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
}
// PublishUnique publishes a raw message on the bus, deduplicated by key.
func (b *TaskBus) PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error {
return b.transport.PublishUnique(ctx, subject, idempotencyKey, payload)
}
// Request sends a request on the bus and waits for its reply.
func (b *TaskBus) Request(ctx context.Context, subject string, payload []byte) ([]byte, error) {
return b.transport.Request(ctx, subject, payload)
}
// InFlight counts the same submitted work that graceful shutdown waits for,
// including tasks waiting for a repository or concurrency slot.
func (d *Daemon) InFlight() int {
dispatcher := d.taskDispatcher()
dispatcher.mu.Lock()
defer dispatcher.mu.Unlock()
return dispatcher.pending
func (b *TaskBus) InFlight() int {
return b.dispatcher.inFlight()
}
// taskDispatcher limits concurrent tasks globally and runs one task per repo
@@ -347,7 +406,7 @@ func (d *taskDispatcher) finishRun(hold *taskHold) {
}
// setCallWaits applies a durable snapshot without accumulating duplicate events.
func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits int) {
d.mu.Lock()
if d.holds[hold.id] != hold || hold.finished {
d.mu.Unlock()
@@ -363,7 +422,7 @@ func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
d.slotsFree.Broadcast()
} else if previous > 0 {
// Reacquiring may block behind another task; the caller keeps running.
go d.reacquire(hold)
go d.reacquire(ctx, hold)
}
}
@@ -371,8 +430,9 @@ func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
// and a slot under the global limit. The run keeps executing meanwhile -- a
// wait on a callee is not a pause. A hold records one slot and one place, so a
// re-acquire queued while another is still waiting gives back whichever of the
// two it finds already recorded instead of taking a second.
func (d *taskDispatcher) reacquire(hold *taskHold) {
// two it finds already recorded instead of taking a second. Once ctx is done
// it stops waiting and gives back whatever the hold owns.
func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) {
d.mu.Lock()
if hold.finished || hold.waits > 0 {
// The run returned, or started waiting again, after the reconciliation that
@@ -391,7 +451,15 @@ func (d *taskDispatcher) reacquire(hold *taskHold) {
d.mu.Unlock()
if previous != nil {
<-previous
select {
case <-previous:
case <-ctx.Done():
d.mu.Lock()
d.giveBackLocked(hold)
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
}
d.mu.Lock()
@@ -421,7 +489,8 @@ func (d *taskDispatcher) reacquire(hold *taskHold) {
// Submit queues a task's process behind the global limit and, unless the repo
// allows concurrency, behind the task already holding that repo. process is
// handed the run's capacity so it can name it once it knows the task's id and
// give it back while it waits on a callee.
// give it back while it waits on a callee. A run whose ctx ends while it waits
// for its repo's place returns without running process.
func (d *taskDispatcher) Submit(
ctx context.Context,
task *workflow.Task,
@@ -446,7 +515,12 @@ func (d *taskDispatcher) Submit(
go func() {
defer d.finishRun(hold)
if previous != nil {
<-previous
select {
case <-previous:
case <-ctx.Done():
d.leaveQueue()
return
}
}
d.mu.Lock()
d.takeSlotLocked()
@@ -457,6 +531,13 @@ func (d *taskDispatcher) Submit(
}()
}
// leaveQueue drops a submitted run that was cancelled before it took a slot.
func (d *taskDispatcher) leaveQueue() {
d.mu.Lock()
d.waiting--
d.mu.Unlock()
}
// hasRoom reports whether a newly submitted task could take a slot without
// queueing behind work already submitted. A run waiting on a callee holds no
// slot, so its callee still finds room.
+13 -9
View File
@@ -24,8 +24,8 @@ import (
// them then never reaches a nil branch in the code under test. A test that
// does exercise one sets its own fake.
func wired(d *Daemon) *Daemon {
if d.Tasks == nil {
d.Tasks = neutralTaskBus{}
if d.TaskTransport == nil {
d.TaskTransport = neutralTaskTransport{}
}
if d.Store == nil {
d.Store = neutralStore{}
@@ -72,6 +72,9 @@ func wired(d *Daemon) *Daemon {
if d.Log == nil {
d.Log = slog.New(slog.DiscardHandler)
}
if d.Cfg == nil {
d.Cfg = config.NewHolder(config.Config{})
}
if d.BindingDispatch == nil {
d.BindingDispatch = NewBindingDispatch(BindingDispatchDeps{
Store: d.Store, Mappings: neutralMappings{}, Bindings: neutralBindings{},
@@ -84,11 +87,12 @@ func wired(d *Daemon) *Daemon {
Log: d.Log,
})
}
d.WireTaskBus()
d.IssuePoller = NewIssuePoller(d.Cfg, d.Store, d.Log,
func(context.Context) []IssueSource { return nil },
func(context.Context, int64, string) error { return nil },
d.PublishTask, d, d.BindingDispatch)
d.ReviewScanner = NewReviewScanner(d.Store, neutralReactions{}, d.Tasks.PublishUnique, d.Bus, d, d, d, d.Log)
d.TaskBus.PublishTask, d, d.BindingDispatch)
d.ReviewScanner = NewReviewScanner(d.Store, neutralReactions{}, d.TaskTransport.PublishUnique, d.Bus, d, d, d, d.Log)
return d
}
@@ -115,16 +119,16 @@ type neutralCreator struct {
storecontract.BindingTaskCreator
}
// neutralTaskBus takes no work and accepts every publish.
type neutralTaskBus struct{}
// neutralTaskTransport takes no work and accepts every publish.
type neutralTaskTransport struct{}
func (neutralTaskBus) PublishUnique(context.Context, string, string, []byte) error { return nil }
func (neutralTaskTransport) PublishUnique(context.Context, string, string, []byte) error { return nil }
func (neutralTaskBus) Fetch(context.Context) (eventbus.Message, error) {
func (neutralTaskTransport) Fetch(context.Context) (eventbus.Message, error) {
return nil, eventbus.ErrNoMessage
}
func (neutralTaskBus) Request(context.Context, string, []byte) ([]byte, error) { return nil, nil }
func (neutralTaskTransport) Request(context.Context, string, []byte) ([]byte, error) { return nil, nil }
// neutralStore records no events. Any other store call panics, so a test that
// reaches one sets its own store.