diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index ba64cdd37..18a24e039 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -592,7 +592,7 @@ func (b *boot) issueForge(ctx context.Context, owner, repo string) (forge.IssueF func (b *boot) buildDaemon(ctx context.Context) { log := b.log - b.d = &daemon.Daemon{ + d := &daemon.Daemon{ ResolveRepository: b.repositoryAccess, Cfg: b.cfgHolder, ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken, Scoped: b.cfg.NATS.Mode != config.NATSModeExternal}, @@ -609,7 +609,7 @@ func (b *boot) buildDaemon(ctx context.Context) { PackageAuthorities: installedAuthorities{b.stateStoreGrants.Client}, Storage: b.storeBackend, Log: log, - Tasks: b.natsClient, + TaskTransport: b.natsClient, RunCredentials: b.stateStoreGrants, ContainerPool: b.containerPool, ModelEgress: b.modelEgress, @@ -621,9 +621,9 @@ func (b *boot) buildDaemon(ctx context.Context) { Playbooks: b.livePlaybooks(), } - b.d.Access = b.accessChain - b.d.ReviewScanner = daemon.NewReviewScanner(b.stateStore, b.reactionClient, b.natsClient.PublishUnique, b.bus, b.d, b.d, b.d, log) - b.d.BindingDispatch = daemon.NewBindingDispatch(daemon.BindingDispatchDeps{ + d.Access = b.accessChain + d.ReviewScanner = daemon.NewReviewScanner(b.stateStore, b.reactionClient, b.natsClient.PublishUnique, b.bus, d, d, d, log) + d.BindingDispatch = daemon.NewBindingDispatch(daemon.BindingDispatchDeps{ Store: b.stateStore, Mappings: b.stateClient, Bindings: b.stateClient, @@ -635,11 +635,13 @@ func (b *boot) buildDaemon(ctx context.Context) { Access: b.accessChain, Denials: b.stateClient, Cfg: b.cfgHolder, - Orgs: b.d, + Orgs: d, OrgPrincipals: b.orgPrincipals, Log: log, }) - b.d.IssuePoller = daemon.NewIssuePoller(b.cfgHolder, b.stateStore, log, b.issueSources, b.taskActions().Withdraw, b.d.PublishTask, b.d, b.d.BindingDispatch) + d.WireTaskBus() + d.IssuePoller = daemon.NewIssuePoller(b.cfgHolder, b.stateStore, log, b.issueSources, b.taskActions().Withdraw, d.TaskBus.PublishTask, d, d.BindingDispatch) + b.d = d } func (b *boot) publishConfig(ctx context.Context, cfg config.Config) { diff --git a/internal/app/archied/control_plane.go b/internal/app/archied/control_plane.go index 0d113feb4..db2a90247 100644 --- a/internal/app/archied/control_plane.go +++ b/internal/app/archied/control_plane.go @@ -405,7 +405,7 @@ func (b *boot) applyRuntimeResourceUpdate(ctx context.Context, kind string, upda // dispatcher is created later from the config the next drain reads. func (b *boot) resizeTaskDispatcher(maxConcurrency int) { if b.d != nil { - b.d.ResizeTaskDispatcher(maxConcurrency) + b.d.TaskBus.ResizeTaskDispatcher(maxConcurrency) } } diff --git a/internal/app/archied/health.go b/internal/app/archied/health.go index 1a6212a2c..aed54153b 100644 --- a/internal/app/archied/health.go +++ b/internal/app/archied/health.go @@ -94,7 +94,7 @@ func (b *boot) startHealth(ctx context.Context) error { if b.d == nil { return -1 } - return b.d.InFlight() + return b.d.TaskBus.InFlight() }, } addr := b.cfg.Health.Listen diff --git a/internal/daemon/agent_status_test.go b/internal/daemon/agent_status_test.go index 8a519778d..460bc4888 100644 --- a/internal/daemon/agent_status_test.go +++ b/internal/daemon/agent_status_test.go @@ -65,10 +65,10 @@ func TestRunViaAgentRequiresTerminalStatus(t *testing.T) { } store := &statusStore{} d := wired(&Daemon{ - Cfg: config.NewHolder(config.Config{}), - Store: store, - Tasks: statusBus{reply: reply}, - Log: slog.New(slog.DiscardHandler), + Cfg: config.NewHolder(config.Config{}), + Store: store, + TaskTransport: statusBus{reply: reply}, + Log: slog.New(slog.DiscardHandler), }) d.runViaAgent(t.Context(), &workflow.Task{ID: 7, Attempt: 1, Owner: "owner", Repo: "repo", Workflow: "implement"}, config.Repo{}, config.AgentProfile{}, "") diff --git a/internal/daemon/call_waits.go b/internal/daemon/call_waits.go index 80f81c277..23fa60082 100644 --- a/internal/daemon/call_waits.go +++ b/internal/daemon/call_waits.go @@ -25,7 +25,7 @@ func (d *Daemon) watchCallWaits(ctx context.Context) { return } sub := d.Bus.Subscribe(callWaitBuffer) - dispatcher := d.taskDispatcher() + bus := d.TaskBus go func() { defer sub.Close() ticker := time.NewTicker(time.Second) @@ -39,29 +39,29 @@ func (d *Daemon) watchCallWaits(ctx context.Context) { return } if e.Kind == events.KindWorkflowCallStarted || e.Kind == events.KindWorkflowCallFinished || e.Kind == events.KindStageStart || e.Kind == events.KindStageFinish { - d.reconcileCallWaits(ctx, dispatcher, reader) + bus.reconcileCallWaits(ctx, reader) } case <-ticker.C: - d.reconcileCallWaits(ctx, dispatcher, reader) + bus.reconcileCallWaits(ctx, reader) } } }() } // Events only prompt a read; periodic reads recover dropped starts and finishes. -func (d *Daemon) reconcileCallWaits(ctx context.Context, dispatcher *taskDispatcher, reader storecontract.StepReader) { - dispatcher.mu.Lock() - holds := make([]*taskHold, 0, len(dispatcher.holds)) - for _, hold := range dispatcher.holds { +func (b *TaskBus) reconcileCallWaits(ctx context.Context, reader storecontract.StepReader) { + b.dispatcher.mu.Lock() + holds := make([]*taskHold, 0, len(b.dispatcher.holds)) + for _, hold := range b.dispatcher.holds { holds = append(holds, hold) } - dispatcher.mu.Unlock() + b.dispatcher.mu.Unlock() for _, hold := range holds { readCtx, cancel := context.WithTimeout(ctx, 5*time.Second) steps, err := reader.ListSteps(readCtx, hold.id, hold.attempt) cancel() if err != nil { - d.Log.WarnContext(ctx, "read call waits", "task", hold.id, "err", err) + b.log.WarnContext(ctx, "read call waits", "task", hold.id, "err", err) continue } waits := 0 @@ -70,6 +70,6 @@ func (d *Daemon) reconcileCallWaits(ctx context.Context, dispatcher *taskDispatc waits++ } } - dispatcher.setCallWaits(hold, waits) + b.dispatcher.setCallWaits(ctx, hold, waits) } } diff --git a/internal/daemon/container_run.go b/internal/daemon/container_run.go index fa2b6376d..c68144561 100644 --- a/internal/daemon/container_run.go +++ b/internal/daemon/container_run.go @@ -306,7 +306,7 @@ func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) deadline := time.Now().Add(taskRunReadyTimeout) backoff := taskRunRetryBackoff for { - reply, err := d.Tasks.Request(ctx, subject, data) + reply, err := d.TaskBus.Request(ctx, subject, data) if err == nil { return reply, nil } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 2e6d7285a..a6fd6592c 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -29,9 +29,9 @@ import ( "github.com/samcharles93/archie-core/internal/worktree" ) -// TaskBus publishes discovered work, collects it back, and asks a worker to -// run a task. -type TaskBus interface { +// TaskTransport publishes discovered work, collects it back, and asks a worker +// to run a task. The daemon's TaskBus runs on it. +type TaskTransport interface { PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error Fetch(ctx context.Context) (eventbus.Message, error) Request(ctx context.Context, subject string, payload []byte) ([]byte, error) @@ -100,8 +100,11 @@ type Daemon struct { Trees *worktree.Manager Bus *events.Bus Log *slog.Logger - // Tasks is the NATS task bus. - Tasks TaskBus + // TaskTransport is the NATS connection the task bus runs on. The bus is + // built by WireTaskBus. + TaskTransport TaskTransport + // TaskBus takes, runs and publishes tasks. Set by WireTaskBus. + TaskBus *TaskBus // ReviewScanner reconciles PRs, scans their reviews and drains review // reactions into remediate runs. ReviewScanner *ReviewScanner @@ -125,12 +128,6 @@ type Daemon struct { // execution is unavailable and process parks tasks; every runnable task gets // a fresh container. ContainerPool *container.Pool - // dispatcher bounds task execution globally across poll passes. Built once, - // on first use, so ResizeTaskDispatcher can apply a live - // containers.max_concurrency change without a restart; drainNATS submits - // each pass through it and leaves its tasks running. - dispatcherOnce sync.Once - dispatcher *taskDispatcher // Storage is the pluggable storage backend for container mounts. // A runnable task requires it; acquireTaskContainer parks on nil. Storage storage.Backend diff --git a/internal/daemon/dispatcher_test.go b/internal/daemon/dispatcher_test.go index f2755ebd2..e69177f58 100644 --- a/internal/daemon/dispatcher_test.go +++ b/internal/daemon/dispatcher_test.go @@ -150,7 +150,7 @@ func changeCallWaits(d *taskDispatcher, delta int, id int64) { } d.mu.Unlock() if hold != nil { - d.setCallWaits(hold, waits) + d.setCallWaits(context.Background(), hold, waits) } } @@ -339,7 +339,7 @@ func TestSecondReacquireDoesNotQueueASecondPlace(t *testing.T) { // A second re-acquire for the same run, with the first still waiting. A // Resume cannot queue this one: queueing two takes a Suspend between them, // and that Suspend would clear the place queued above. - go d.reacquire(taskHoldOf(d, 1)) + go d.reacquire(context.Background(), taskHoldOf(d, 1)) awaitNoState(t, "a second place was queued for the run", func() bool { return placeClaim(d, 1) != queued }) awaitState(t, "task 1 did not take its slot back", func() bool { return holdsCapacity(d, 1) }) @@ -436,7 +436,7 @@ func TestCallWaitsRecoverDroppedStart(t *testing.T) { cfg := config.Config{} cfg.Containers.MaxConcurrency = 1 d := wired(&Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)}) - dispatcher := d.taskDispatcher() + dispatcher := d.TaskBus.dispatcher started := make(chan int64, 2) a, b := make(chan struct{}), make(chan struct{}) defer func() { close(a); dispatcher.Wait() }() @@ -526,15 +526,15 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) { }, } d := wired(&Daemon{ - Cfg: config.NewHolder(config.Config{}), - Log: slog.New(slog.DiscardHandler), - Store: store, - Tasks: stubTasks{}, + Cfg: config.NewHolder(config.Config{}), + Log: slog.New(slog.DiscardHandler), + Store: store, + TaskTransport: stubTasks{}, }) drained := make(chan struct{}) go func() { - d.drainNATS(context.Background()) + d.TaskBus.drainNATS(context.Background()) close(drained) }() @@ -552,7 +552,7 @@ func TestDrainPassDoesNotWaitForItsTasks(t *testing.T) { // Shutdown is where that wait now lives. waited := make(chan struct{}) go func() { - d.WaitForTasks() + d.TaskBus.WaitForTasks() close(waited) }() select { @@ -605,13 +605,13 @@ func TestDrainPassClaimsABacklogWithoutWaitingPerTask(t *testing.T) { } tasks := &countingTasks{} d := wired(&Daemon{ - Cfg: config.NewHolder(config.Config{}), - Log: slog.New(slog.DiscardHandler), - Store: store, - Tasks: tasks, + Cfg: config.NewHolder(config.Config{}), + Log: slog.New(slog.DiscardHandler), + Store: store, + TaskTransport: tasks, }) - d.drainNATS(context.Background()) - d.WaitForTasks() + d.TaskBus.drainNATS(context.Background()) + d.TaskBus.WaitForTasks() if len(store.queue) != 0 { t.Fatalf("%d tasks left unclaimed", len(store.queue)) } @@ -633,13 +633,52 @@ func TestDrainPassClaimsOnlyWhatTheDispatcherCanRun(t *testing.T) { cfg := config.Config{} cfg.Containers.MaxConcurrency = 2 d := wired(&Daemon{ - Cfg: config.NewHolder(cfg), - Log: slog.New(slog.DiscardHandler), - Store: store, - Tasks: &countingTasks{}, + Cfg: config.NewHolder(cfg), + Log: slog.New(slog.DiscardHandler), + Store: store, + TaskTransport: &countingTasks{}, }) - d.drainNATS(context.Background()) + d.TaskBus.drainNATS(context.Background()) if got := len(store.queue); got != 8 { t.Fatalf("pass left %d tasks unclaimed, want 8", got) } } + +// A run queued behind another in its repo's chain returns once its ctx is +// cancelled, so shutdown's wait is not held by a run that never gets a turn. +// Before the fix it waited on the chain forever and left pending and waiting +// counted. +func TestCancelledRunQueuedBehindItsRepoReturns(t *testing.T) { + d := newTaskDispatcher(2, nil) + started := make(chan int64, 4) + a := make(chan struct{}) + + submitRun(d, 1, "app", started, a) + awaitStart(t, started, 1) + + ctx, cancel := context.WithCancel(t.Context()) + d.Submit(ctx, &workflow.Task{ID: 2, Owner: "acme", Repo: "app"}, func(context.Context, *workflow.Task, *taskHold) { + started <- 2 + }) + awaitNoStart(t, started, "while task 1 held the repo's place") + cancel() + awaitState(t, "the cancelled run is still counted as pending", func() bool { return d.inFlight() == 1 }) + + close(a) + waited := make(chan struct{}) + go func() { + d.Wait() + close(waited) + }() + select { + case <-waited: + case <-time.After(runDeadline): + t.Fatal("Wait did not return: the cancelled run never gave up its place") + } + awaitNoStart(t, started, "after its ctx was cancelled") + d.mu.Lock() + defer d.mu.Unlock() + if d.pending != 0 || d.waiting != 0 { + t.Fatalf("pending = %d, waiting = %d after every run returned, want 0 and 0", d.pending, d.waiting) + } +} diff --git a/internal/daemon/intake_cap_test.go b/internal/daemon/intake_cap_test.go index a09965304..9254e87ed 100644 --- a/internal/daemon/intake_cap_test.go +++ b/internal/daemon/intake_cap_test.go @@ -55,7 +55,7 @@ func TestPublishDiscoveredCap(t *testing.T) { for _, n := range tt.existing { have[n] = true } - d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, Tasks: bus}) + d := wired(&Daemon{Log: slog.New(slog.DiscardHandler), Store: taskLookup{have: have}, TaskTransport: bus}) var issues []forge.Issue for n := 1; n <= tt.issues; n++ { issues = append(issues, forge.Issue{Number: n, Labels: []string{"archie"}}) diff --git a/internal/daemon/intake_disabled_test.go b/internal/daemon/intake_disabled_test.go index ecb96f918..c87c27bef 100644 --- a/internal/daemon/intake_disabled_test.go +++ b/internal/daemon/intake_disabled_test.go @@ -91,7 +91,7 @@ func TestPollNATSSkipsDisabledRoutedWorkflow(t *testing.T) { fg := &reactCounter{} var logs bytes.Buffer d := wired(&Daemon{ - Log: slog.New(slog.NewTextHandler(&logs, nil)), Store: taskLookup{}, Tasks: bus, + Log: slog.New(slog.NewTextHandler(&logs, nil)), Store: taskLookup{}, TaskTransport: bus, WorkflowDefinitions: tt.definitions, Principals: tt.principals, }) diff --git a/internal/daemon/lifecycle.go b/internal/daemon/lifecycle.go index d58db7d40..7cd81ef95 100644 --- a/internal/daemon/lifecycle.go +++ b/internal/daemon/lifecycle.go @@ -81,13 +81,13 @@ const sweepTimeout = 2 * time.Minute func (d *Daemon) Run(ctx context.Context) error { // A pass no longer waits for the tasks it submitted, so the run does, once // the ctx is cancelled. - defer d.WaitForTasks() + defer d.TaskBus.WaitForTasks() d.watchCallWaits(ctx) drained := make(chan struct{}) defer func() { <-drained }() go func() { defer close(drained) - d.drainLoop(ctx) + d.TaskBus.drainLoop(ctx, d.BindingDispatch.dispatchBindings) }() interval := d.Cfg.Get().PollInterval.Std() ticker := time.NewTicker(interval) @@ -112,7 +112,7 @@ func (d *Daemon) Run(ctx context.Context) error { // on a callee stays in flight while the callee sits queued, so the queue is // drained again until every task, callees included, has returned. func (d *Daemon) RunOnce(ctx context.Context) { - defer d.WaitForTasks() + defer d.TaskBus.WaitForTasks() d.watchCallWaits(ctx) // One pass has no scheduler, so it runs the maintenance jobs itself. for _, job := range d.BuiltinJobs() { @@ -122,15 +122,15 @@ func (d *Daemon) RunOnce(ctx context.Context) { } d.BindingDispatch.dispatchBindings(ctx) d.Cycle(ctx) - d.drainNATS(ctx) + d.TaskBus.drainNATS(ctx) ticker := time.NewTicker(time.Second) defer ticker.Stop() - for d.taskDispatcher().inFlight() > 0 { + for d.TaskBus.InFlight() > 0 { select { case <-ctx.Done(): return case <-ticker.C: - d.drainNATS(ctx) + d.TaskBus.drainNATS(ctx) } } } diff --git a/internal/daemon/model_egress_test.go b/internal/daemon/model_egress_test.go index 9d13d57b9..f9e188515 100644 --- a/internal/daemon/model_egress_test.go +++ b/internal/daemon/model_egress_test.go @@ -96,8 +96,8 @@ func TestRunViaAgentCarriesProviderCredentialVariable(t *testing.T) { "used": {Class: "deepseek", APIKey: config.SecretRef{Engine: "env", Key: "USED_KEY"}}, }, }), - Tasks: bus, - Log: slog.New(slog.DiscardHandler), + TaskTransport: bus, + Log: slog.New(slog.DiscardHandler), }) task := &workflow.Task{ID: 7, Identity: string(sys), Attempt: 1, Workflow: "implement", WorkflowDefinitionYAML: "id: w\nsteps:\n - type: agent.run\n"} diff --git a/internal/daemon/poll_identity_test.go b/internal/daemon/poll_identity_test.go index d8a4f88d1..39bd177bf 100644 --- a/internal/daemon/poll_identity_test.go +++ b/internal/daemon/poll_identity_test.go @@ -55,7 +55,7 @@ func TestPolledTaskKeepsItsCreatingIdentity(t *testing.T) { t.Run(tt.name, func(t *testing.T) { bus := &publishedKeys{} d := wired(&Daemon{ - Log: slog.New(slog.DiscardHandler), Store: taskLookup{}, Tasks: bus, + Log: slog.New(slog.DiscardHandler), Store: taskLookup{}, TaskTransport: bus, Cfg: config.NewHolder(config.Config{}), }) d.IssuePoller.sources = func(context.Context) []IssueSource { diff --git a/internal/daemon/task_bus.go b/internal/daemon/task_bus.go index d7fd9217c..0f212c320 100644 --- a/internal/daemon/task_bus.go +++ b/internal/daemon/task_bus.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" "sync" "time" "github.com/samcharles93/archie-core/internal/domain/identity" + "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/eventbus" @@ -17,18 +19,79 @@ import ( "github.com/samcharles93/archie-core/internal/taskstate" ) +// TaskBus takes work from the NATS stream and the store's requeue fallback, +// runs it under one global dispatcher, and publishes discovered work. +type TaskBus struct { + transport TaskTransport + store storecontract.TaskStore + identities identity.Repository + log *slog.Logger + orgs orgResolver + process func(context.Context, *workflow.Task, *taskHold) + emit func(context.Context, events.Event) + markPass func(loop string) + dispatcher *taskDispatcher +} + +// TaskBusDeps are the inputs a TaskBus runs on. Every field is required. +type TaskBusDeps struct { + Transport TaskTransport + Store storecontract.TaskStore + Identities identity.Repository + Log *slog.Logger + Orgs orgResolver + MaxConcurrency int + AllowConcurrent func(*workflow.Task) bool + Process func(context.Context, *workflow.Task, *taskHold) + Emit func(context.Context, events.Event) + MarkPass func(loop string) +} + +// NewTaskBus builds a bus whose dispatcher starts at MaxConcurrency. +func NewTaskBus(deps TaskBusDeps) *TaskBus { + return &TaskBus{ + transport: deps.Transport, + store: deps.Store, + identities: deps.Identities, + log: deps.Log, + orgs: deps.Orgs, + process: deps.Process, + emit: deps.Emit, + markPass: deps.MarkPass, + dispatcher: newTaskDispatcher(deps.MaxConcurrency, deps.AllowConcurrent), + } +} + +// WireTaskBus builds the daemon's task bus from its fields. The composition +// root calls it once the fields are set, and before the daemon is published: +// process is unexported, so only this package can name it. +func (d *Daemon) WireTaskBus() { + d.TaskBus = NewTaskBus(TaskBusDeps{ + Transport: d.TaskTransport, + Store: d.Store, + Identities: d.IdentityRepository, + Log: d.Log, + Orgs: d, + MaxConcurrency: d.Cfg.Get().Containers.MaxConcurrency, + AllowConcurrent: d.allowConcurrentForTask, + Process: d.process, + Emit: d.emit, + MarkPass: d.markPass, + }) +} + // drainLoop is the only task drainer and capture dispatcher while Run is // active. It runs apart from the poll tick so a captured event becomes a // running task within about a second regardless of poll_interval or forge // latency. Submit never blocks, so a full dispatcher // just leaves tasks waiting on slots and the loop stays paced by the ticker. -func (d *Daemon) drainLoop(ctx context.Context) { +func (b *TaskBus) drainLoop(ctx context.Context, dispatchBindings func(context.Context)) { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { - d.markPass("drain") - d.BindingDispatch.dispatchBindings(ctx) - d.drainNATS(ctx) + b.markPass("drain") + dispatchBindings(ctx) + b.drainNATS(ctx) select { case <-ctx.Done(): return @@ -43,19 +106,18 @@ func (d *Daemon) drainLoop(ctx context.Context) { // are both empty: in-flight tasks keep running, so a task waiting on a callee // does not hold up the pass that would claim it. Shutdown waits for them // through WaitForTasks. -func (d *Daemon) drainNATS(ctx context.Context) { - dispatcher := d.taskDispatcher() +func (b *TaskBus) drainNATS(ctx context.Context) { // A full dispatcher leaves work in the stream and the store, not parked // on slots where it already reads as running. - for ctx.Err() == nil && dispatcher.hasRoom() { - msg, err := d.Tasks.Fetch(ctx) + for ctx.Err() == nil && b.dispatcher.hasRoom() { + msg, err := b.transport.Fetch(ctx) if err != nil && !errors.Is(err, eventbus.ErrNoMessage) { - d.Log.ErrorContext(ctx, "nats fetch failed", "err", err) + b.log.ErrorContext(ctx, "nats fetch failed", "err", err) break } if err == nil { - d.submitNATSTask(ctx, dispatcher, msg) - d.markPass("drain") + b.submitNATSTask(ctx, msg) + b.markPass("drain") continue } @@ -63,7 +125,7 @@ func (d *Daemon) drainNATS(ctx context.Context) { // the SQLite fallback must run: requeued tasks (waiting_approval // approval, retry-parked) never went through a NATS publish, so // nothing will ever arrive on the stream to trigger them. - d.claimRequeued(ctx, dispatcher) + b.claimRequeued(ctx) return } } @@ -72,134 +134,124 @@ func (d *Daemon) drainNATS(ctx context.Context) { // dispatcher has room; the rest stay unclaimed for a later pass. It does // not go back to the stream between claims: each empty fetch waits a full poll // timeout, so a backlog left by a restart would outlast the systemd watchdog. -func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) { - for ctx.Err() == nil && dispatcher.hasRoom() { - task, err := d.Store.ClaimNext(ctx) +func (b *TaskBus) claimRequeued(ctx context.Context) { + for ctx.Err() == nil && b.dispatcher.hasRoom() { + task, err := b.store.ClaimNext(ctx) if err != nil { - d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err) + b.log.ErrorContext(ctx, "sqlite claim failed", "err", err) return } if task == nil { return } - dispatcher.Submit(ctx, task, d.process) - d.markPass("drain") + b.dispatcher.Submit(ctx, task, b.process) + b.markPass("drain") } } -// taskDispatcher returns the daemon's long-lived global dispatcher, building -// it from the running config on first use. It outlives a single drain pass so -// ResizeTaskDispatcher can resize the running dispatcher. -func (d *Daemon) taskDispatcher() *taskDispatcher { - d.dispatcherOnce.Do(func() { - d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask) - }) - return d.dispatcher -} - // WaitForTasks blocks until every submitted task's process has returned. A // drain pass no longer waits for its own tasks, so shutdown is where in-flight // work is waited for. -func (d *Daemon) WaitForTasks() { - d.taskDispatcher().Wait() +func (b *TaskBus) WaitForTasks() { + b.dispatcher.Wait() } // ResizeTaskDispatcher applies a new containers.max_concurrency to the // running global task dispatcher without a restart. Running tasks keep their // slots; a lowered limit is honoured as they finish, and a raised limit // admits queued work immediately. -func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) { - d.taskDispatcher().SetMaxConcurrency(maxConcurrency) +func (b *TaskBus) ResizeTaskDispatcher(maxConcurrency int) { + b.dispatcher.SetMaxConcurrency(maxConcurrency) } // submitNATSTask decodes a fetched message and queues it for processing. A // message that cannot be decoded is acked rather than redelivered: it will // never decode on a retry, so leaving it unacked would block the queue. -func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) { +func (b *TaskBus) submitNATSTask(ctx context.Context, msg eventbus.Message) { envelope, err := workintake.DecodeTask(msg.Data()) if err != nil { - d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err) + b.log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err) if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err) + b.log.WarnContext(ctx, "nats ack failed", "subject", msg.Subject(), "err", err) } return } task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity} - dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) { - d.processNATSTask(ctx, msg, hold) + b.dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) { + b.processNATSTask(ctx, msg, hold) }) } // processNATSTask decodes a NATS message, writes it to SQLite, claims it, // and runs the workflow. The message is ack'd on terminal (park is a valid // outcome). Nak on transient errors so NATS redelivers. -func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) { +func (b *TaskBus) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) { ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header()) defer span.End() tm, err := workintake.DecodeTask(msg.Data()) if err != nil { - d.Log.ErrorContext(ctx, "nats decode failed", "err", err) + b.log.ErrorContext(ctx, "nats decode failed", "err", err) if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) + b.log.WarnContext(ctx, "ack failed", "err", err) } // bad message, don't retry return } - if !d.acceptTaskPublisher(ctx, msg, tm.Identity) { + if !b.acceptTaskPublisher(ctx, msg, tm.Identity) { return } - inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity) + inserted, err := b.store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity) if err != nil { - d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err) + b.log.ErrorContext(ctx, "nats enqueue failed", "err", err) if err := msg.Nak(); err != nil { - d.Log.WarnContext(ctx, "nats nak failed", "err", err) + b.log.WarnContext(ctx, "nats nak failed", "err", err) } return } if !inserted { // Already tracked -- dedup. Ack and move on. if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) + b.log.WarnContext(ctx, "ack failed", "err", err) } return } - task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number) + task, err := b.store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number) if err != nil { - d.Log.ErrorContext(ctx, "nats claim failed", "err", err) + b.log.ErrorContext(ctx, "nats claim failed", "err", err) if err := msg.Nak(); err != nil { - d.Log.WarnContext(ctx, "nats nak failed", "err", err) + b.log.WarnContext(ctx, "nats nak failed", "err", err) } return } if task == nil { // Claimed by another consumer, or task is not queued. Ack. if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) + b.log.WarnContext(ctx, "ack failed", "err", err) } return } - d.emitTaskQueued(ctx, task) - d.process(ctx, task, hold) + b.emitTaskQueued(ctx, task) + b.process(ctx, task, hold) if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) + b.log.WarnContext(ctx, "ack failed", "err", err) } } -func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool { - if ok, reason, class := d.identityMayAct(ctx, name); !ok { - d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject()) +func (b *TaskBus) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool { + if ok, reason, class := b.orgs.identityMayAct(ctx, name); !ok { + b.log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject()) if class == taskstate.ParkTransient { if err := msg.Nak(); err != nil { - d.Log.WarnContext(ctx, "nak failed", "err", err) + b.log.WarnContext(ctx, "nak failed", "err", err) } return false } if err := msg.Ack(); err != nil { - d.Log.WarnContext(ctx, "ack failed", "err", err) + b.log.WarnContext(ctx, "ack failed", "err", err) } return false } @@ -209,28 +261,28 @@ func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, // emitTaskQueued records the queued task under the identity that polled it, // which is both the actor and the authority the task was created with. -func (d *Daemon) emitTaskQueued(ctx context.Context, task *workflow.Task) { +func (b *TaskBus) emitTaskQueued(ctx context.Context, task *workflow.Task) { e := events.Event{ Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo, Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt, ActorID: task.Identity, PrincipalID: task.Identity, } - if value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(task.Identity)); err == nil { + if value, err := b.identities.Get(ctx, identity.IdentityID(task.Identity)); err == nil { e.ActorKind = string(value.Kind) } - d.emit(ctx, e) + b.emit(ctx, e) } // PublishTask validates a task envelope, stamps the producing identity's // org and publishes it. The idempotency key dedups rediscovered issues. -func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error { +func (b *TaskBus) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error { if err := task.Kind.Validate(); err != nil { return fmt.Errorf("publish task %s: %w", task.Ref(), err) } if task.Identity == "" { return fmt.Errorf("publish task %s: no identity", task.Ref()) } - resolved, err := d.identityOrg(ctx, identity.IdentityID(task.Identity)) + resolved, err := b.orgs.identityOrg(ctx, identity.IdentityID(task.Identity)) if err != nil { return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err) } @@ -239,16 +291,23 @@ func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) if err != nil { return err } - return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload) + return b.transport.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload) +} + +// PublishUnique publishes a raw message on the bus, deduplicated by key. +func (b *TaskBus) PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error { + return b.transport.PublishUnique(ctx, subject, idempotencyKey, payload) +} + +// Request sends a request on the bus and waits for its reply. +func (b *TaskBus) Request(ctx context.Context, subject string, payload []byte) ([]byte, error) { + return b.transport.Request(ctx, subject, payload) } // InFlight counts the same submitted work that graceful shutdown waits for, // including tasks waiting for a repository or concurrency slot. -func (d *Daemon) InFlight() int { - dispatcher := d.taskDispatcher() - dispatcher.mu.Lock() - defer dispatcher.mu.Unlock() - return dispatcher.pending +func (b *TaskBus) InFlight() int { + return b.dispatcher.inFlight() } // taskDispatcher limits concurrent tasks globally and runs one task per repo @@ -347,7 +406,7 @@ func (d *taskDispatcher) finishRun(hold *taskHold) { } // setCallWaits applies a durable snapshot without accumulating duplicate events. -func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) { +func (d *taskDispatcher) setCallWaits(ctx context.Context, hold *taskHold, waits int) { d.mu.Lock() if d.holds[hold.id] != hold || hold.finished { d.mu.Unlock() @@ -363,7 +422,7 @@ func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) { d.slotsFree.Broadcast() } else if previous > 0 { // Reacquiring may block behind another task; the caller keeps running. - go d.reacquire(hold) + go d.reacquire(ctx, hold) } } @@ -371,8 +430,9 @@ func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) { // and a slot under the global limit. The run keeps executing meanwhile -- a // wait on a callee is not a pause. A hold records one slot and one place, so a // re-acquire queued while another is still waiting gives back whichever of the -// two it finds already recorded instead of taking a second. -func (d *taskDispatcher) reacquire(hold *taskHold) { +// two it finds already recorded instead of taking a second. Once ctx is done +// it stops waiting and gives back whatever the hold owns. +func (d *taskDispatcher) reacquire(ctx context.Context, hold *taskHold) { d.mu.Lock() if hold.finished || hold.waits > 0 { // The run returned, or started waiting again, after the reconciliation that @@ -391,7 +451,15 @@ func (d *taskDispatcher) reacquire(hold *taskHold) { d.mu.Unlock() if previous != nil { - <-previous + select { + case <-previous: + case <-ctx.Done(): + d.mu.Lock() + d.giveBackLocked(hold) + d.mu.Unlock() + d.slotsFree.Broadcast() + return + } } d.mu.Lock() @@ -421,7 +489,8 @@ func (d *taskDispatcher) reacquire(hold *taskHold) { // Submit queues a task's process behind the global limit and, unless the repo // allows concurrency, behind the task already holding that repo. process is // handed the run's capacity so it can name it once it knows the task's id and -// give it back while it waits on a callee. +// give it back while it waits on a callee. A run whose ctx ends while it waits +// for its repo's place returns without running process. func (d *taskDispatcher) Submit( ctx context.Context, task *workflow.Task, @@ -446,7 +515,12 @@ func (d *taskDispatcher) Submit( go func() { defer d.finishRun(hold) if previous != nil { - <-previous + select { + case <-previous: + case <-ctx.Done(): + d.leaveQueue() + return + } } d.mu.Lock() d.takeSlotLocked() @@ -457,6 +531,13 @@ func (d *taskDispatcher) Submit( }() } +// leaveQueue drops a submitted run that was cancelled before it took a slot. +func (d *taskDispatcher) leaveQueue() { + d.mu.Lock() + d.waiting-- + d.mu.Unlock() +} + // hasRoom reports whether a newly submitted task could take a slot without // queueing behind work already submitted. A run waiting on a callee holds no // slot, so its callee still finds room. diff --git a/internal/daemon/wiring_test.go b/internal/daemon/wiring_test.go index 46a676648..01e3d4800 100644 --- a/internal/daemon/wiring_test.go +++ b/internal/daemon/wiring_test.go @@ -24,8 +24,8 @@ import ( // them then never reaches a nil branch in the code under test. A test that // does exercise one sets its own fake. func wired(d *Daemon) *Daemon { - if d.Tasks == nil { - d.Tasks = neutralTaskBus{} + if d.TaskTransport == nil { + d.TaskTransport = neutralTaskTransport{} } if d.Store == nil { d.Store = neutralStore{} @@ -72,6 +72,9 @@ func wired(d *Daemon) *Daemon { if d.Log == nil { d.Log = slog.New(slog.DiscardHandler) } + if d.Cfg == nil { + d.Cfg = config.NewHolder(config.Config{}) + } if d.BindingDispatch == nil { d.BindingDispatch = NewBindingDispatch(BindingDispatchDeps{ Store: d.Store, Mappings: neutralMappings{}, Bindings: neutralBindings{}, @@ -84,11 +87,12 @@ func wired(d *Daemon) *Daemon { Log: d.Log, }) } + d.WireTaskBus() d.IssuePoller = NewIssuePoller(d.Cfg, d.Store, d.Log, func(context.Context) []IssueSource { return nil }, func(context.Context, int64, string) error { return nil }, - d.PublishTask, d, d.BindingDispatch) - d.ReviewScanner = NewReviewScanner(d.Store, neutralReactions{}, d.Tasks.PublishUnique, d.Bus, d, d, d, d.Log) + d.TaskBus.PublishTask, d, d.BindingDispatch) + d.ReviewScanner = NewReviewScanner(d.Store, neutralReactions{}, d.TaskTransport.PublishUnique, d.Bus, d, d, d, d.Log) return d } @@ -115,16 +119,16 @@ type neutralCreator struct { storecontract.BindingTaskCreator } -// neutralTaskBus takes no work and accepts every publish. -type neutralTaskBus struct{} +// neutralTaskTransport takes no work and accepts every publish. +type neutralTaskTransport struct{} -func (neutralTaskBus) PublishUnique(context.Context, string, string, []byte) error { return nil } +func (neutralTaskTransport) PublishUnique(context.Context, string, string, []byte) error { return nil } -func (neutralTaskBus) Fetch(context.Context) (eventbus.Message, error) { +func (neutralTaskTransport) Fetch(context.Context) (eventbus.Message, error) { return nil, eventbus.ErrNoMessage } -func (neutralTaskBus) Request(context.Context, string, []byte) ([]byte, error) { return nil, nil } +func (neutralTaskTransport) Request(context.Context, string, []byte) ([]byte, error) { return nil, nil } // neutralStore records no events. Any other store call panics, so a test that // reaches one sets its own store.