Files
archie/internal/daemon/daemon.go
T
sam 3af96fd818
Quality / check (push) Canceled after 26s
deploy / gate (push) Successful in 4m30s
deploy / archied-image (push) Successful in 2m50s
deploy / agent-image (push) Successful in 12s
deploy / dist-zip (push) Skipped
fix(daemon): watch the task intake loop too and name the stalled loop
2026-11-02 00:26:50 +11:00

2727 lines
101 KiB
Go

// Package daemon is archied's resident loop: poll GitHub for labelled
// issues, enqueue them, and process tasks through their routed workflows.
// State lives in the store; the daemon is restartable at any point.
package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"log/slog"
"maps"
"os"
"path/filepath"
"strings"
"sync"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/agentrun"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/eda/playbook"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/eventbus"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
"github.com/samcharles93/archie-core/internal/logging"
"github.com/samcharles93/archie-core/internal/natsrpc"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/taskrun"
"github.com/samcharles93/archie-core/internal/taskstate"
"github.com/samcharles93/archie-core/internal/worktree"
)
// TaskBus publishes discovered work, collects it back, and asks a worker to
// run a task.
type TaskBus interface {
PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error
Fetch(ctx context.Context) (eventbus.Message, error)
Request(ctx context.Context, subject string, payload []byte) ([]byte, error)
}
// RunCredentialIssuer issues the one credential a task's container holds: it
// authorizes the task's own State Store calls and the push of its branch.
// It lapses at the task's time limit plus a margin if never revoked.
type RunCredentialIssuer interface {
Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error)
}
// NATSEndpoint is the broker address and credential archied connected with at
// startup. It is runtime state rather than configuration: embedded mode
// generates both values, and external secret references are resolved once.
type NATSEndpoint struct {
URL string
Token string
// Scoped is true on the embedded broker, where a container logs in with
// its run credential and reaches only its own task's subjects.
Scoped bool
}
// ContainerToken is the broker token a container is handed: none on a
// scoped broker. An external broker cannot scope a container, so it still
// gets the broker's token (reported as a health issue by archied).
func (e NATSEndpoint) ContainerToken() string {
if e.Scoped {
return ""
}
return e.Token
}
// StateStoreEndpoint is the State Store target and the CA its certificate
// chains to, both passed to agent containers.
type StateStoreEndpoint struct {
URL string
CA []byte
}
type Daemon struct {
ResolveRepository func(context.Context, *workflow.Task, string) (RepositoryAccess, error)
// Cfg publishes the running configuration. Read through Cfg() so a
// reload swaps the published snapshot atomically. See config.Holder.
Cfg *config.Holder
// ConnectedNATS is the endpoint the daemon connected with at startup;
// container env uses it, never live config.
ConnectedNATS NATSEndpoint
// ConnectedStateStore is the State Store target the daemon dialed at
// startup, passed to containers.
ConnectedStateStore StateStoreEndpoint
Store storecontract.TaskStore
// Mappings persists payload field mappings.
// Used by the binding dispatch loop to resolve capture bodies against
// the mapping a binding names. Optional: nil disables the binding
// dispatch loop (legacy behaviour).
Mappings storecontract.MappingStore
// Bindings persists playbook bindings.
// Optional: nil disables the binding dispatch loop (legacy behaviour).
Bindings storecontract.BindingStore
// BindingDispatcher lists undispatched captures, finds armed bindings and
// records dispatches. Nil disables binding dispatch.
BindingDispatcher storecontract.BindingDispatcher
// MappingMatches counts each event a mapping resolved at dispatch.
// Optional: nil leaves match counts unrecorded.
MappingMatches storecontract.MappingMatchRecorder
// BindingTaskCreator enqueues a task triggered by a binding. Split
// off TaskLifecycle so the lifecycle surface stays narrow and the
// dispatch loop does not acquire the full task-creation contract.
// Optional: nil disables the dispatch loop.
BindingTaskCreator storecontract.BindingTaskCreator
// Access evaluates the policy chain at dispatch, the second of the two
// Authorizer call sites: the workflow's identity may `run` the workflow in
// its workspace. Optional: nil dispatches without the chain, which is the
// behaviour of an install that has not built it.
Access access.Authorizer
// Principals assembles the dispatch principal for the workflow's
// identity. Wired with Access or not at all.
Principals access.PrincipalSource
// Denials records dispatch refusals. Optional: nil skips the record.
Denials access.DenialRecorder
Forge forge.Forge
Trees *worktree.Manager
Bus *events.Bus
Log *slog.Logger
// Tasks is the NATS task bus. Always set in production.
Tasks TaskBus
// reactionConsumer is built lazily by drainReactions and kept for its
// lifetime drop counter.
reactionConsumer *reactionConsumer
// lastPolled remembers each repo's eligible issues from its previous
// complete poll, under the dispatch rule that produced them, so only an
// issue that was eligible and dropped out reads as withdrawn.
lastPolledMu sync.Mutex
lastPolled map[string]polledIssues
// reactionPublisher is built lazily by scanPRReviews; an indirection
// only so tests can swap the delivery without the bus.
reactionPublisher ReactionPublisher
// Reactions is the review-reaction stream, drained into remediate runs each
// cycle. Nil disables it.
Reactions ReactionSource
// RunCredentials issues each task's run credential (see runCredential).
// Required whenever ConnectedStateStore.URL is set: the task is parked
// rather than handed the daemon's own administrative token.
RunCredentials RunCredentialIssuer
// KitLauncher starts tasks whose agent profile is a Kit. Nil parks them.
KitLauncher KitLauncher
// ModelEgress gives native task containers model access without provider
// keys. Nil parks a task whose providers need a key.
ModelEgress ModelEgress
// TaskModels answers the model aliases a task's org resolves against.
// Nil runs every task on the instance aliases.
TaskModels TaskModels
// TaskInstructions answers the instance and organisation instructions a
// task's agent runs follow. Nil runs without them.
TaskInstructions TaskInstructions
// TaskRunReadyTimeout bounds how long runViaAgent retries a taskrun request
// while the new container has not subscribed yet. Zero uses
// defaultTaskRunReadyTimeout.
TaskRunReadyTimeout time.Duration
// TaskRunRetryBackoff is the delay between retry attempts within
// TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff.
TaskRunRetryBackoff time.Duration
// AgentStatus records agent versions for update verification. Nil disables
// it.
AgentStatus *AgentStatus
// ContainerPool manages Docker container lifecycle. Nil means autonomous
// execution is unavailable and process parks tasks; every runnable task gets
// a fresh container.
ContainerPool *container.Pool
// dispatcher bounds task execution globally across poll passes. Built once,
// on first use, so ResizeTaskDispatcher can apply a live
// containers.max_concurrency change without a restart; drainNATS submits
// each pass through it and leaves its tasks running.
dispatcherOnce sync.Once
dispatcher *taskDispatcher
// Storage is the pluggable storage backend for container mounts.
// A runnable task requires it; acquireTaskContainer parks on nil.
Storage storage.Backend
// IssueSources resolves, each poll cycle, every organisation connection
// to poll and the identity whose permit reaches it. Nil polls nothing.
IssueSources func(context.Context) []IssueSource
// OrgPrincipals is one acting principal per organisation, system org
// first. Nil dispatches the root identity's org alone.
OrgPrincipals func(context.Context) []access.Principal
// IdentityRepository is the durable lifecycle authority. Polling checks it
// every cycle so suspend/reactivate commands do not require a restart.
IdentityRepository identity.Repository
AgentAssignments org.AgentAssignments
RootIdentityID identity.IdentityID
// PackageAuthorities bounds the tools a package workflow's run is offered.
PackageAuthorities PackageAuthorities
// Playbooks are the loaded EDA playbooks, consulted before kind/label
// bindings when pinning a workflow. Nil means none.
Playbooks interface {
Dispatch(context.Context, playbook.DispatchInput) (playbook.Decision, bool)
Run(context.Context, storecontract.PlaybookDispatcher, *slog.Logger, playbook.DispatchInput) error
}
// PlaybookLedger is the at-most-once gate action playbooks run through.
// Nil means action playbooks do not run: they never fire unguarded.
PlaybookLedger storecontract.PlaybookDispatcher
// WorkflowDefinitions supplies the active database definitions. A task is
// pinned once before dispatch; retries reuse the task's stored YAML.
WorkflowDefinitions interface {
WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error)
}
// WithdrawTask declines a task whose issue a complete poll no longer
// returns. Nil leaves withdrawn issues' work running.
WithdrawTask func(ctx context.Context, taskID int64, reason string) error
// WorkflowEnablement supplies which workflows each org has disabled. A
// binding targeting a workflow its org disabled does not dispatch. Nil
// means every workflow is enabled.
WorkflowEnablement interface {
WorkflowEnablement(context.Context) (workflowtask.WorkflowEnablement, error)
}
// TaskLogs persists each task's log output and mirrors it to the dashboard.
// Nil disables task logging.
TaskLogs *logging.TaskRegistry
// running holds a cancel function for every task currently executing,
// so /stop can reach work already in flight. Its zero value is ready
// to use.
running runningTasks
// loopPasses maps each run loop ("cycle", "drain") to when its latest
// pass began, as Unix nanoseconds. StalestLoop reports the oldest, so one
// wedged loop isn't hidden by a healthy one.
loopPasses sync.Map
}
// IssueSource is one forge connection's repositories as the identity
// permitted to read them sees them.
type IssueSource struct {
Identity identity.IdentityID
Forge forge.Forge
// BotUser is the connection's external account, the assignee polled for.
BotUser string
// Dispatch and Label are the discovery rule the connection's org polls
// with. Empty means the daemon's own.
Dispatch config.Dispatch
Label string
Repos []config.Repo
}
func (d *Daemon) issueSources(ctx context.Context) []IssueSource {
if d.IssueSources == nil {
return nil
}
return d.IssueSources(ctx)
}
// sourceTag is the identity a source's tasks carry: empty for the root.
func (d *Daemon) sourceTag(source IssueSource) string {
if source.Identity == d.RootIdentityID {
return ""
}
return string(source.Identity)
}
// emit publishes an observability event; safe on a nil bus.
func (d *Daemon) emit(ctx context.Context, e events.Event) {
if d.Bus != nil {
d.Bus.Publish(ctx, e)
}
}
// Startup runs crash recovery and access verification once.
func (d *Daemon) Startup(ctx context.Context) error {
if err := d.cleanupExpiredStorage(ctx); err != nil {
d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err)
}
n, err := d.Store.RecoverStale(ctx)
if err != nil {
return err
}
if n > 0 {
d.Log.InfoContext(ctx, "re-queued tasks left running by a previous daemon", "count", n)
}
d.sweepAccess(ctx)
return nil
}
// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and
// the run loop, and a forge client carries no request deadline of its own, so
// an unreachable host would otherwise hold boot open indefinitely.
const sweepTimeout = 2 * time.Minute
// sweepAccess accepts pending invitations and verifies push access through
// every organisation connection, each in its own goroutine.
func (d *Daemon) sweepAccess(ctx context.Context) {
sweep := func(source IssueSource) {
log := d.Log.With("identity", source.Identity)
ctx, cancel := context.WithTimeout(ctx, sweepTimeout)
defer cancel()
if err := source.Forge.AcceptInvitations(ctx, repoRefs(source.Repos)); err != nil {
log.WarnContext(ctx, "invitation sweep failed", "err", err)
}
for _, r := range source.Repos {
if err := source.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil {
// A sweep cut short by its own context is not a repo failure: name
// the budget when it expired, stay quiet on a shutdown.
switch {
case errors.Is(ctx.Err(), context.DeadlineExceeded):
// Every later call would return the same error, so say it once.
log.WarnContext(ctx, "push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
return
case ctx.Err() != nil:
return
}
log.WarnContext(ctx, "repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
}
}
}
var wg sync.WaitGroup
for _, source := range d.issueSources(ctx) {
wg.Go(func() { sweep(source) })
}
wg.Wait()
}
// repoRefs is a source's configured repositories as the allowlist the forge
// accepts invitations against.
func repoRefs(repos []config.Repo) []forge.RepoRef {
out := make([]forge.RepoRef, 0, len(repos))
for _, r := range repos {
out = append(out, forge.RepoRef{Owner: r.Owner, Repo: r.Name})
}
return out
}
// Run polls until the context ends.
func (d *Daemon) Run(ctx context.Context) error {
// A pass no longer waits for the tasks it submitted, so the run does, once
// the ctx is cancelled.
defer d.WaitForTasks()
d.watchCallWaits(ctx)
if d.Tasks != nil {
drained := make(chan struct{})
defer func() { <-drained }()
go func() {
defer close(drained)
d.drainLoop(ctx)
}()
}
interval := d.Cfg.Get().PollInterval.Std()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
d.Cycle(ctx)
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
// PollInterval may have been reloaded; reset the ticker so a
// change takes effect on the next cycle.
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
interval = iv
ticker.Reset(iv)
}
}
}
}
// drainLoop is the only task drainer while Run is active. It runs apart from
// the poll tick so a queued task starts within about a second regardless of
// poll_interval or forge latency. Submit never blocks, so a full dispatcher
// just leaves tasks waiting on slots and the loop stays paced by the ticker.
func (d *Daemon) drainLoop(ctx context.Context) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
d.markPass("drain")
d.drainNATS(ctx)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// pollIssuesWithConfig returns the repo's issues the dispatch rule makes
// work, using the given forge client and dispatch config. complete is false when any part of the poll failed or was skipped, so
// the result must not be read as the whole eligible set.
func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) {
// Webhook intake hears about issues as they change; polling as well would
// spend forge API quota to find the same work. The receiver runs from the
// root forge settings, so their mode decides for every identity.
if d.Cfg.Get().Forge.Intake == config.ForgeIntakeWebhook {
return nil, false
}
labelled := func() ([]forge.Issue, bool) {
// Empty-label rule shared with workintake.MatchesDispatch: a label
// trigger needs a non-empty label or it would match every open issue.
if cfg.Label == "" {
d.Log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName())
return nil, false
}
issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
if err != nil {
d.Log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err)
return nil, false
}
return issues, true
}
assigned := func() ([]forge.Issue, bool) {
if cfg.BotUser == "" {
d.Log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName())
return nil, false
}
issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
if err != nil {
d.Log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err)
return nil, false
}
return issues, true
}
switch cfg.Dispatch.Trigger {
case "label":
return labelled()
case "either":
byAssignee, assignedOK := assigned()
byLabel, labelledOK := labelled()
seen := map[int]bool{}
for _, is := range byAssignee {
seen[is.Number] = true
}
issues = byAssignee
for _, is := range byLabel {
if !seen[is.Number] {
issues = append(issues, is)
}
}
return issues, assignedOK && labelledOK
default: // "assignee" (default)
return assigned()
}
}
// RunOnce runs one cycle and drains until no task is in flight. A run waiting
// on a callee stays in flight while the callee sits queued, so the queue is
// drained again until every task, callees included, has returned.
func (d *Daemon) RunOnce(ctx context.Context) {
defer d.WaitForTasks()
d.watchCallWaits(ctx)
// One pass has no scheduler, so it runs the maintenance jobs itself.
for _, job := range d.BuiltinJobs() {
if err := job.Run(ctx); err != nil {
d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err)
}
}
d.Cycle(ctx)
if d.Tasks == nil {
return
}
d.drainNATS(ctx)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for d.taskDispatcher().inFlight() > 0 {
select {
case <-ctx.Done():
return
case <-ticker.C:
d.drainNATS(ctx)
}
}
}
// Cycle is one poll pass: turn captured events and review reactions into
// tasks; maintenance runs as built-in schedules. Draining queued tasks is the caller's job
// (drainLoop in Run, drainNATS in RunOnce) so there is one drainer.
func (d *Daemon) Cycle(ctx context.Context) {
d.markPass("cycle")
d.dispatchBindings(ctx)
d.drainReactions(ctx)
}
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
if ttl <= 0 || d.Storage == nil {
return nil
}
n, err := d.Storage.CleanupExpired(ctx, ttl)
if n > 0 {
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
}
if err != nil {
return fmt.Errorf("persistent volume cleanup: %w", err)
}
return nil
}
// bindingDispatchBatchLimit caps how many undispatched captures a single
// dispatchBindings pass will fetch from the store. The cap is defensive
// (the captures table is also pruned by retention/maxEvents) and bounds a
// single cycle's worst-case latency on a backlog.
const bindingDispatchBatchLimit = 100
// dispatchBindings offers each identified capture to every armed binding on
// its source. Nil Bindings, BindingDispatcher or BindingTaskCreator disables
// it.
func (d *Daemon) dispatchBindings(ctx context.Context) {
if d.Bindings == nil || d.BindingDispatcher == nil || d.BindingTaskCreator == nil {
return
}
if d.Principals == nil {
d.dispatchOrgBindings(ctx)
return
}
var principals []access.Principal
if d.OrgPrincipals != nil {
principals = d.OrgPrincipals(ctx)
} else {
principal, err := d.Principals.PrincipalFor(ctx, d.RootIdentityID)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: principal unavailable", "identity", d.RootIdentityID, "err", err)
return
}
principals = []access.Principal{principal}
}
seen := make(map[org.OrgID]bool)
for _, principal := range principals {
if seen[principal.Org] {
continue
}
seen[principal.Org] = true
d.dispatchOrgBindings(org.WithOrg(access.WithPrincipal(ctx, principal), principal.Org))
}
}
func (d *Daemon) dispatchOrgBindings(ctx context.Context) {
// Gather the armed bindings whose workflow is enabled first, so the
// oldest-first captures query is not held by captures that only a binding
// on a disabled workflow would take.
bindings, err := d.Bindings.ListBindings(ctx)
if err != nil {
d.Log.WarnContext(ctx, "list bindings failed", "error", err)
return
}
enablement, err := d.workflowEnablement(ctx)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: workflow enablement unavailable", "error", err)
return
}
enabled := make([]string, 0, len(bindings))
for _, b := range bindings {
if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) {
enabled = append(enabled, b.ID)
}
}
if len(enabled) == 0 {
return
}
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit)
if err != nil {
d.Log.WarnContext(ctx, "list undispatched captures failed", "error", err)
return
}
workflows, err := d.activeWorkflows(ctx)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: workflow definitions unavailable", "error", err)
return
}
for _, c := range captures {
d.dispatchCapture(ctx, c, workflows, enablement)
}
}
// dispatchCapture offers one capture to every armed binding on its source. A
// disabled workflow's binding is skipped like an unarmed one, so the capture
// waits rather than recording a failure every cycle.
func (d *Daemon) dispatchCapture(ctx context.Context, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection, enablement workflowtask.WorkflowEnablement) {
armed, err := d.BindingDispatcher.ArmedBindingsForSource(ctx, c.Source)
if err != nil {
d.Log.WarnContext(ctx, "armed bindings lookup", "source", c.Source, "error", err)
return
}
for _, b := range armed {
if b.OrgID == org.OrgFromContext(ctx) && b.Matcher.Matches(c.Source, c.Dispatchable()) && enablement.Enabled(b.OrgID, b.Workflow) {
d.dispatchOneBinding(ctx, b, c, workflows)
}
}
}
// bindingMapping returns b's mapping, or nil after logging why it is
// unavailable. A mapping that no longer exists is a terminal outcome.
func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent) *mapping.Mapping {
if d.Mappings == nil {
d.Log.WarnContext(ctx, "binding dispatch: mapping store unavailable", "binding", b.ID)
return nil
}
m, err := d.Mappings.GetMapping(ctx, b.MappingID)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err)
return nil
}
if m == nil {
d.recordDispatchFailure(ctx, b, c, fmt.Sprintf("mapping %q does not exist", b.MappingID), nil)
}
return m
}
// dispatchOneBinding offers one capture to one binding: the mapping must
// match the capture's event type and resolve required fields, the filter
// must admit the values, and the workflow must accept the inputs. The
// ledger claim and the task commit together, so each binding fires exactly
// once per delivery.
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) {
m := d.bindingMapping(ctx, b, c)
if m == nil || m.EventTypeID != c.EventType {
return
}
values, failures := mapping.Resolve(m.Fields, []byte(c.Body))
if hasBlockingFailure(m.Fields, failures) {
d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures})
return
}
if d.MappingMatches != nil {
if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil {
d.Log.WarnContext(ctx, "binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err)
}
}
filter, err := binding.CompileFilter(b.Filter, m.Fields)
if err != nil {
d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()})
return
}
if admitted, _ := filter.Admits(values); !admitted {
d.recordEvaluated(ctx, b, c, "filtered")
return
}
target, reason, ok := d.resolveBindingTarget(b, values, workflows)
if !ok {
d.recordDispatchFailure(ctx, b, c, reason, nil)
return
}
// The chain decides dispatch: the binding's principal may `run` this
// workflow in its workspace, and the event's signature result and address
// travel with the request. A denial is terminal; an unavailable principal
// or policy chain leaves the capture listed for the next cycle.
if !d.authorizeDispatch(ctx, b, c, target) {
return
}
d.claimAndEnqueue(ctx, b, c, target, values)
}
// claimAndEnqueue asks the State Store to claim the dispatch and enqueue its
// task as one unit. A failure leaves the delivery undispatched, so the next
// cycle offers it again; a delivery already dispatched is refused.
func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) {
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
body := renderBindingBody(values, c)
if target.declared {
body = fmt.Sprintf("Started by binding %q from capture %q; the workflow's inputs travel with the task.", b.Name, c.ID)
}
taskIdentity := ""
if principal, ok := access.PrincipalFromContext(ctx); ok && (principal.IdentityID != d.RootIdentityID || principal.Org != org.DefaultOrgID) {
taskIdentity = string(principal.IdentityID)
}
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs)
if err != nil {
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
d.Log.WarnContext(ctx, "binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
}
return
}
d.markBindingStart(ctx, task.ID, b, c)
}
// authorizeDispatch checks the binding's principal, its org's dispatching
// identity, may start the workflow. A denial is recorded and reported as a
// binding_dispatch_failure. The run itself acts as the workflow's declared
// identity, which adoptWorkflowIdentity checks when the task is processed.
func (d *Daemon) authorizeDispatch(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget) bool {
if d.Access == nil || d.Principals == nil {
return true
}
principal, ok := access.PrincipalFromContext(ctx)
if !ok {
d.Log.WarnContext(ctx, "binding dispatch: no principal; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID)
return false
}
resource := access.Resource{
Kind: access.KindWorkflow, ID: b.Workflow, Org: principal.Org, Workspace: b.WorkspaceID,
}
contextValue := access.Context{Signature: signatureOf(c), Addr: c.RemoteAddr}
decision := d.Access.Authorize(principal, access.ActionRun, resource, contextValue)
if decision.Allowed {
return true
}
if errors.Is(decision.Err, access.ErrChainUnavailable) {
// Nothing was evaluated. Recording it would mark the pair evaluated for
// good, so a State Store blip would drop the capture instead of leaving
// it for the next cycle.
d.Log.WarnContext(ctx, "binding dispatch: access chain unavailable; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID, "err", decision.Err)
return false
}
if d.Denials != nil {
denial := access.Denial{
Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun,
Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level,
Policies: decision.Policies,
}
if err := d.Denials.RecordDenial(ctx, denial); err != nil {
d.Log.WarnContext(ctx, "binding dispatch: record denial", "binding", b.ID, "error", err)
}
}
d.recordDispatchFailure(ctx, b, c, "access denied", map[string]any{
"level": string(decision.Level), "policies": decision.Policies,
})
return false
}
// signatureOf names the event's signature result as the dispatch context
// carries it: a signed event and an approved unsigned event are the two
// dispatchable shapes (storecontract.CapturedEvent.Dispatchable).
func signatureOf(c storecontract.CapturedEvent) string {
switch {
case c.Authenticated:
return "valid"
case c.Unsigned:
return "unsigned"
}
return "unverified"
}
// markBindingStart records on a task's timeline the binding and captured
// event that started it.
func (d *Daemon) markBindingStart(ctx context.Context, taskID int64, b binding.Binding, c storecontract.CapturedEvent) {
if _, err := d.Store.InsertEvent(ctx, events.Event{
TaskID: taskID,
Kind: events.KindBindingStarted,
Detail: fmt.Sprintf("started by binding %q from source %q", b.Name, c.Source),
Data: map[string]any{
"binding_id": b.ID, "binding_name": b.Name, "binding_version": b.Version,
"capture_id": c.ID, "source": c.Source, "unsigned": c.Unsigned,
},
}); err != nil {
d.Log.WarnContext(ctx, "binding dispatch: start marker", "task", taskID, "error", err)
}
}
// recordDispatchFailure records why a binding did not dispatch a capture and
// marks the pair evaluated, so the capture is not offered to it again.
func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) {
d.recordEvaluated(ctx, b, c, reason)
data := map[string]any{
"binding_id": b.ID,
"binding_version": b.Version,
"capture_id": c.ID,
}
maps.Copy(data, extra)
_, _ = d.Store.InsertEvent(ctx, events.Event{
Kind: "binding_dispatch_failure",
Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason),
Data: data,
})
}
// recordEvaluated writes the ledger row for an evaluation that started no
// task. Only terminal outcomes call it; transient ones leave the capture
// listed for the next cycle.
func (d *Daemon) recordEvaluated(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string) {
err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, reason)
if err != nil && !errors.Is(err, storecontract.ErrAlreadyDispatched) {
d.Log.WarnContext(ctx, "binding dispatch: record evaluation", "binding", b.ID, "capture", c.ID, "error", err)
}
}
// hasBlockingFailure reports whether any failure is on a required field.
func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool {
if len(failures) == 0 {
return false
}
required := make(map[string]bool, len(fields))
for _, f := range fields {
if f.Required {
required[f.Name] = true
}
}
for _, fail := range failures {
if required[fail.FieldName] {
return true
}
}
return false
}
// renderBindingBody builds the task body from the values a mapping
// resolved, one "key=value" pair per line. Empty maps still produce
// a body that records the source capture id so the workflow has
// provenance to hand back to the operator.
func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string {
if len(values) == 0 {
return fmt.Sprintf("(no fields resolved from capture %q)", c.ID)
}
parts := make([]string, 0, len(values))
for k, v := range values {
parts = append(parts, fmt.Sprintf("%s=%v", k, v))
}
return strings.Join(parts, "\n")
}
// resolveBindingRepo returns the binding's pinned owner/repo, or the only
// configured repo. With zero or several repos and no pin it refuses.
func (d *Daemon) resolveBindingRepo(b binding.Binding) (owner, repo, reason string) {
if b.Owner != "" && b.Repo != "" {
return b.Owner, b.Repo, ""
}
repos := d.Cfg.Get().Repos
if len(repos) == 1 {
return repos[0].Owner, repos[0].Name, ""
}
return "", "", fmt.Sprintf("cannot pick a repository: %d are configured; pin the binding to one", len(repos))
}
// drainReactions turns available review reactions into queued remediate
// runs. It runs before the task drain so a reaction queued in this pass can
// be claimed by it; the consumer is built lazily once, so its drop counter
// spans the process lifetime.
func (d *Daemon) drainReactions(ctx context.Context) {
if d.Reactions == nil || d.Store == nil {
return
}
if d.reactionConsumer == nil {
d.reactionConsumer = newReactionConsumer(
// The store's owned-PR lookup is the authorization boundary: a
// reaction only ever causes work against a PR archie opened and
// still owns (pr-review-remediation.md decision 3).
d.Store,
d.Store,
d.Store,
d.botUserForTask,
d.Log,
)
}
if _, err := d.reactionConsumer.drain(ctx, d.Reactions, maxReactionsPerCycle); err != nil {
d.Log.ErrorContext(ctx, "reaction drain failed", "err", err)
}
}
// botUserForTask resolves the bot login that owns task, so the reaction
// consumer can exclude Archie's own comments: the global forge bot user for
// root tasks.
func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
if !d.rootTask(task) {
return ""
}
return d.Cfg.Get().BotUser
}
// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for
// requeued tasks (waiting_approval approval, retry-parked) that didn't come
// through a NATS publish. It returns once the stream and the claim fallback
// are both empty: in-flight tasks keep running, so a task waiting on a callee
// does not hold up the pass that would claim it. Shutdown waits for them
// through WaitForTasks.
func (d *Daemon) drainNATS(ctx context.Context) {
dispatcher := d.taskDispatcher()
for ctx.Err() == nil {
msg, err := d.Tasks.Fetch(ctx)
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
d.Log.ErrorContext(ctx, "nats fetch failed", "err", err)
break
}
if err == nil {
d.submitNATSTask(ctx, dispatcher, msg)
continue
}
// ErrNoMessage means the stream is drained. That is precisely when
// the SQLite fallback must run: requeued tasks (waiting_approval
// approval, retry-parked) never went through a NATS publish, so
// nothing will ever arrive on the stream to trigger them.
task, claimErr := d.Store.ClaimNext(ctx)
if claimErr != nil {
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", claimErr)
break
}
if task == nil {
break
}
dispatcher.Submit(ctx, task, d.process)
}
}
// taskDispatcher returns the daemon's long-lived global dispatcher, building
// it from the running config on first use. It outlives a single drain pass so
// ResizeTaskDispatcher can resize the running dispatcher.
func (d *Daemon) taskDispatcher() *taskDispatcher {
d.dispatcherOnce.Do(func() {
d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask)
})
return d.dispatcher
}
// WaitForTasks blocks until every submitted task's process has returned. A
// drain pass no longer waits for its own tasks, so shutdown is where in-flight
// work is waited for.
func (d *Daemon) WaitForTasks() {
d.taskDispatcher().Wait()
}
// ResizeTaskDispatcher applies a new containers.max_concurrency to the
// running global task dispatcher without a restart. Running tasks keep their
// slots; a lowered limit is honoured as they finish, and a raised limit
// admits queued work immediately.
func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) {
d.taskDispatcher().SetMaxConcurrency(maxConcurrency)
}
// submitNATSTask decodes a fetched message and queues it for processing. A
// message that cannot be decoded is acked rather than redelivered: it will
// never decode on a retry, so leaving it unacked would block the queue.
func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) {
envelope, err := workintake.DecodeTask(msg.Data())
if err != nil {
d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err)
_ = msg.Ack()
return
}
task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) {
d.processNATSTask(ctx, msg, hold)
})
}
// taskDispatcher limits concurrent tasks globally and runs one task per repo
// at a time, in submit order, unless the repo allows concurrency.
type taskDispatcher struct {
allowConcurrent func(task *workflow.Task) bool
mu sync.Mutex
limit int
active int
slotsFree *sync.Cond
repoTail map[string]chan struct{}
pending int
// holds maps a task id to the capacity its run holds, so a run waiting on
// a callee can give that capacity back and take it again by task id.
holds map[int64]*taskHold
}
// taskHold is one submitted task's claim on execution capacity: its slot under
// the global limit, and its place at the tail of its repo's chain, which the
// next task for that repo is blocked on. A run that waits on a callee gives
// both back so the callee can run, and takes them again when the wait ends.
type taskHold struct {
dispatcher *taskDispatcher
repo string
// chain is true when the repo runs one task at a time, so the run holds a
// place in that repo's chain.
chain bool
// baton is the run's place in its repo's chain, closed when it gives that
// place up; nil while it holds none.
baton chan struct{}
// slot is true while the run counts against the global limit.
slot bool
// waits counts the callees the run is waiting on: an open call gives capacity
// back and the last closed call takes it again.
waits int
// finished is true once the run's process returned.
finished bool
// id is the task this hold belongs to, named by the run once it knows the
// task's id.
id int64
attempt int
}
// name records which task's capacity this hold is, so a wait on a callee can
// give it back by task id. Zero names nothing: a task arriving on the NATS
// stream has no id until the store claims its row inside the run.
func (h *taskHold) name(id int64, attempt int) {
if id == 0 {
return
}
h.dispatcher.mu.Lock()
h.id = id
h.attempt = attempt
h.dispatcher.holds[id] = h
h.dispatcher.mu.Unlock()
}
// SetMaxConcurrency accepts a new global limit without a restart. Running
// tasks keep their slots; a lowered limit is enforced as they finish, and a
// raised limit admits queued work immediately.
func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) {
d.mu.Lock()
d.limit = maxConcurrency
d.mu.Unlock()
d.slotsFree.Broadcast()
}
func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher {
if allowConcurrent == nil {
allowConcurrent = func(*workflow.Task) bool { return false }
}
d := &taskDispatcher{
limit: maxConcurrency,
allowConcurrent: allowConcurrent,
repoTail: make(map[string]chan struct{}),
holds: make(map[int64]*taskHold),
}
d.slotsFree = sync.NewCond(&d.mu)
return d
}
// joinRepoLocked appends baton to the tail of repo's chain and returns the
// baton ahead of it, which the joining task is blocked on. Callers hold d.mu.
func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} {
previous := d.repoTail[repo]
d.repoTail[repo] = baton
return previous
}
// takeSlotLocked blocks until the global limit has room, then counts the run
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
// Callers hold d.mu.
func (d *taskDispatcher) takeSlotLocked() {
for d.limit > 0 && d.active >= d.limit {
d.slotsFree.Wait()
}
d.active++
}
// giveBackLocked gives back everything the run holds: its slot under the
// global limit, and its place in its repo's chain, which releases the task
// queued behind it. Callers hold d.mu.
func (d *taskDispatcher) giveBackLocked(hold *taskHold) {
if hold.slot {
d.active--
hold.slot = false
}
if hold.baton == nil {
return
}
close(hold.baton)
if d.repoTail[hold.repo] == hold.baton {
delete(d.repoTail, hold.repo)
}
hold.baton = nil
}
// finishRun retires a returned run: it holds no capacity any more, and no
// Resume may take any for it. Whatever an in-flight resume takes later is
// given back by reacquire, which sees finished.
func (d *taskDispatcher) finishRun(hold *taskHold) {
d.mu.Lock()
hold.finished = true
if hold.id != 0 && d.holds[hold.id] == hold {
delete(d.holds, hold.id)
}
d.giveBackLocked(hold)
d.pending--
d.mu.Unlock()
d.slotsFree.Broadcast()
}
// setCallWaits applies a durable snapshot without accumulating duplicate events.
func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
d.mu.Lock()
if d.holds[hold.id] != hold || hold.finished {
d.mu.Unlock()
return
}
previous := hold.waits
hold.waits = waits
if waits > 0 {
d.giveBackLocked(hold)
}
d.mu.Unlock()
if waits > 0 {
d.slotsFree.Broadcast()
} else if previous > 0 {
// Reacquiring may block behind another task; the caller keeps running.
go d.reacquire(hold)
}
}
// reacquire returns a resumed run's capacity: its place in its repo's chain
// and a slot under the global limit. The run keeps executing meanwhile -- a
// wait on a callee is not a pause. A hold records one slot and one place, so a
// re-acquire queued while another is still waiting gives back whichever of the
// two it finds already recorded instead of taking a second.
func (d *taskDispatcher) reacquire(hold *taskHold) {
d.mu.Lock()
if hold.finished || hold.waits > 0 {
// The run returned, or started waiting again, after the reconciliation that
// queued this: it needs no capacity.
d.mu.Unlock()
return
}
var previous chan struct{}
// A place another re-acquire already queued is the run's one place; joining
// behind it would queue a second, which the task behind it would wait on
// forever.
if hold.chain && hold.baton == nil {
hold.baton = make(chan struct{})
previous = d.joinRepoLocked(hold.repo, hold.baton)
}
d.mu.Unlock()
if previous != nil {
<-previous
}
d.mu.Lock()
d.takeSlotLocked()
if hold.slot {
// Another re-acquire took the run's one slot while this waited: give
// back the slot this one just took. The place is the other's, so it is
// left alone.
d.active--
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
hold.slot = true
if hold.finished || hold.waits > 0 {
// The run ended, or started waiting again, while this was queued for
// capacity: give back what it took, so no slot or place in the repo
// chain leaks.
d.giveBackLocked(hold)
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
d.mu.Unlock()
}
// Submit queues a task's process behind the global limit and, unless the repo
// allows concurrency, behind the task already holding that repo. process is
// handed the run's capacity so it can name it once it knows the task's id and
// give it back while it waits on a callee.
func (d *taskDispatcher) Submit(
ctx context.Context,
task *workflow.Task,
process func(context.Context, *workflow.Task, *taskHold),
) {
repo := task.Owner + "/" + task.Repo
hold := &taskHold{dispatcher: d, repo: repo}
var previous chan struct{}
if !d.allowConcurrent(task) {
hold.chain = true
hold.baton = make(chan struct{})
d.mu.Lock()
previous = d.joinRepoLocked(repo, hold.baton)
d.mu.Unlock()
}
d.mu.Lock()
d.pending++
d.mu.Unlock()
go func() {
defer d.finishRun(hold)
if previous != nil {
<-previous
}
d.mu.Lock()
d.takeSlotLocked()
hold.slot = true
d.mu.Unlock()
process(ctx, task, hold)
}()
}
// inFlight is how many submitted tasks have not returned yet.
func (d *taskDispatcher) inFlight() int {
d.mu.Lock()
defer d.mu.Unlock()
return d.pending
}
func (d *taskDispatcher) Wait() {
d.mu.Lock()
defer d.mu.Unlock()
for d.pending > 0 {
d.slotsFree.Wait()
}
}
// poll discovers work through every organisation connection, resolved
// afresh each cycle so a rotated credential or a new connection applies to
// the next poll.
func (d *Daemon) poll(ctx context.Context) error {
// Webhook intake replaces polling; an empty pass is not a failure.
if d.Cfg.Get().Forge.Intake == config.ForgeIntakeWebhook {
return nil
}
var failed []string
for _, source := range d.issueSources(ctx) {
if !d.identityActive(ctx, source.Identity) {
continue
}
cfg := d.Cfg.Get()
cfg.BotUser = source.BotUser
if source.Dispatch.Trigger != "" {
cfg.Dispatch, cfg.Label = source.Dispatch, source.Label
}
tag := d.sourceTag(source)
for _, repo := range source.Repos {
issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo)
if complete {
d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues)
} else {
failed = append(failed, repo.FullName())
}
for _, is := range issues {
if d.Tasks != nil {
d.pollNATS(ctx, source.Forge, cfg, repo, is, strings.Join(is.Labels, ","), tag)
}
}
}
}
return passFailed("issue poll", failed)
}
// passFailed reports a maintenance pass whose items failed; each item's cause
// is already logged where it happened.
func passFailed(pass string, failed []string) error {
if len(failed) == 0 {
return nil
}
return fmt.Errorf("%s failed for %d item(s): %s", pass, len(failed), strings.Join(failed, ", "))
}
// polledIssues is one repo's eligible issues under one dispatch rule.
type polledIssues struct {
rule string
eligible map[int]bool
}
// rememberPoll records a repo's latest eligible set and returns the previous
// one, reporting whether it was taken under the same rule.
func (d *Daemon) rememberPoll(key string, current polledIssues) (polledIssues, bool) {
d.lastPolledMu.Lock()
defer d.lastPolledMu.Unlock()
previous, seen := d.lastPolled[key]
if d.lastPolled == nil {
d.lastPolled = map[string]polledIssues{}
}
d.lastPolled[key] = current
return previous, seen && previous.rule == current.rule
}
// dispatchRule identifies the rule a poll selects issues by.
func dispatchRule(cfg config.Config) string {
return cfg.Dispatch.Trigger + "\x00" + cfg.Label + "\x00" + cfg.BotUser
}
// withdrawUnpolled declines the queued or running work of an identity's
// issues in repo that a complete poll no longer returns: the issue was
// closed, unlabelled or unassigned.
func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) {
if d.WithdrawTask == nil {
return
}
eligible := make(map[int]bool, len(polled))
for _, is := range polled {
eligible[is.Number] = true
}
// Only an issue the same rule found eligible last time and no longer
// finds was withdrawn. A changed label, trigger or bot user, or the first
// poll after a restart, only records the new set: tasks picked up under
// the old rule were not withdrawn by anyone.
previous, sameRule := d.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible})
if !sameRule {
return
}
active, err := d.Store.TasksPage(ctx, storecontract.TaskPage{
Statuses: []string{taskstate.Queued, taskstate.Running}, Limit: 500,
})
if err != nil {
d.Log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err)
return
}
for _, t := range active {
if t.Owner != repo.Owner || t.Repo != repo.Name || t.Identity != identity ||
!t.IsForgeBacked() || t.PRNumber > 0 || eligible[t.IssueNumber] || !previous.eligible[t.IssueNumber] {
continue
}
if err := d.WithdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil {
d.Log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err)
}
}
}
// pollNATS publishes discovered issues to NATS (new flow).
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) {
// Read-only existence check prevents rediscovering an existing task.
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
if err != nil {
d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return
}
if existing != nil {
return
}
// Classify and encode here: the envelope, its routing kind and its
// subject belong to the work-intake domain, not to the transport.
parsed := workintake.SplitLabels(labels)
task := workintake.TaskEnvelope{
Owner: repo.Owner,
Repo: repo.Name,
Number: is.Number,
Title: is.Title,
Body: is.Body,
Labels: parsed,
Identity: identity,
Kind: workintake.KindForLabels(parsed),
}
if err := d.PublishTask(ctx, task); err != nil {
d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return
}
d.acknowledge(ctx, fg, cfg, repo, is)
}
// PublishTask validates a task envelope, stamps the producing identity's
// org and publishes it. The idempotency key dedups rediscovered issues.
func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
if d.Tasks == nil {
return fmt.Errorf("publish task %s: no task bus configured", task.Ref())
}
if err := task.Kind.Validate(); err != nil {
return fmt.Errorf("publish task %s: %w", task.Ref(), err)
}
resolved, err := d.identityOrg(ctx, d.publisherIdentity(task.Identity))
if err != nil {
return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err)
}
task.Org = resolved
payload, err := task.Encode()
if err != nil {
return err
}
return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
}
// PublishReaction stamps the org on a review reaction, defaulting to the
// root identity's, and publishes it.
func (d *Daemon) PublishReaction(ctx context.Context, reaction workintake.ReviewCommentEnvelope) error {
if d.Tasks == nil {
return fmt.Errorf("publish reaction %s: no task bus configured", reaction.Ref())
}
if reaction.Org == "" {
resolved, err := d.identityOrg(ctx, d.RootIdentityID)
if err != nil {
return fmt.Errorf("publish reaction %s: resolve org: %w", reaction.Ref(), err)
}
reaction.Org = resolved
}
payload, err := reaction.Encode()
if err != nil {
return err
}
return d.Tasks.PublishUnique(ctx, reaction.Subject(), reaction.IdempotencyKey(), payload)
}
// publisherIdentity is the identity a producer publishes as: the envelope's
// identity when it names one, otherwise the root identity.
func (d *Daemon) publisherIdentity(ref string) identity.IdentityID {
if ref == "" || identity.IdentityID(ref) == d.RootIdentityID || identity.StableID(ref) == d.RootIdentityID {
return d.RootIdentityID
}
return identity.IdentityID(ref)
}
// identityOrg resolves an identity's org. No PrincipalSource means the
// default org; a store error is returned.
func (d *Daemon) identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) {
if d.Principals == nil {
return org.DefaultOrgID, nil
}
principal, err := d.Principals.PrincipalFor(ctx, id)
if err != nil {
return "", err
}
return principal.Org, nil
}
// acknowledge posts the pickup reaction and queued event
// using fg -- the forge client that owns repo (identity-scoped or root) --
// and the ack reaction from cfg, the dispatch config that owns this poll.
func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) {
d.Log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title)
if ack := cfg.Dispatch.AckReaction; ack != "" {
if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil {
d.Log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err)
}
}
d.emit(ctx, events.Event{
Kind: events.KindTaskQueued, Repo: repo.FullName(),
Issue: is.Number, Detail: is.Title,
})
}
func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool {
if ok, reason, class := d.identityMayAct(ctx, name); !ok {
d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject())
if class == taskstate.ParkTransient {
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nak failed", "err", err)
}
return false
}
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
return false
}
return true
}
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
// and runs the workflow. The message is ack'd on terminal (park is a valid
// outcome). Nak on transient errors so NATS redelivers.
func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) {
ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header())
defer span.End()
tm, err := workintake.DecodeTask(msg.Data())
if err != nil {
d.Log.ErrorContext(ctx, "nats decode failed", "err", err)
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
// bad message, don't retry
return
}
if !d.acceptTaskPublisher(ctx, msg, tm.Identity) {
return
}
inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
if err != nil {
d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err)
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if !inserted {
// Already tracked -- dedup. Ack and move on.
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
if err != nil {
d.Log.ErrorContext(ctx, "nats claim failed", "err", err)
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if task == nil {
// Claimed by another consumer, or task is not queued. Ack.
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
d.process(ctx, task, hold)
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
}
// closeResolvedIssue closes a finished task's forge issue, skipping chat
// tasks. Failures are logged.
func (d *Daemon) closeResolvedIssue(ctx context.Context, fg forge.Forge, task *workflow.Task, comment string) {
if !task.IsForgeBacked() {
return
}
if d.ResolveRepository != nil {
resolved, err := d.Repository(ctx, task, config.OperationPR)
if err != nil {
d.Log.WarnContext(ctx, "issue closure not permitted", "task", task.ID)
return
}
fg = resolved.Forge
}
if err := fg.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil {
d.Log.WarnContext(ctx, "closing resolved issue failed; it stays open and will be re-polled",
"repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err)
}
}
// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state
// and cleans up their worktrees.
func (d *Daemon) reconcilePRs(ctx context.Context) error {
tasks, err := d.Store.OpenPRs(ctx)
if err != nil {
return fmt.Errorf("reconcile query: %w", err)
}
var failed []string
for _, t := range tasks {
fg := d.forgeFor(ctx, &t)
if _, unavailable := fg.(*forge.NoopForge); unavailable {
continue
}
trees := d.treesFor(&t)
// PRState/PR reconciliation is valid regardless of Source -- a
// chat-sourced task can still open a real PR against a real
// branch. Only the issue-label call (tied to the synthetic
// issue number) is forge-only.
state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber)
if err != nil {
d.Log.WarnContext(ctx, "PR state check failed", "pr", t.PRNumber, "err", err)
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
continue
}
attempt := t.Attempt
switch state {
case "merged":
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, ""); err != nil {
d.Log.ErrorContext(ctx, "PR merged but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
continue
}
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err)
}
// Close the issue explicitly; nothing else does.
d.closeResolvedIssue(ctx, fg, &t, fmt.Sprintf(
"Closed by #%d, merged by archie.", t.PRNumber,
))
d.Log.InfoContext(ctx, "PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
d.emit(ctx, events.Event{
Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt,
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
Data: map[string]any{"pr": t.PRNumber},
})
case "closed":
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge"); err != nil {
d.Log.ErrorContext(ctx, "PR rejected but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
continue
}
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err)
}
d.Log.InfoContext(ctx, "PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
d.emit(ctx, events.Event{
Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt,
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
Data: map[string]any{"pr": t.PRNumber},
})
}
}
return passFailed("PR reconcile", failed)
}
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
// A task off the NATS stream only has an id once the store has claimed its
// row, inside this call, so its capacity is named here.
hold.name(task.ID, task.Attempt)
// The workflow may name the identity the run acts as, which everything
// below is chosen by, so it is settled before anything else.
if !d.adoptWorkflowIdentity(ctx, task) {
return
}
// Register first so the task can be stopped during setup.
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
defer finished()
defer d.openTaskLog(task)()
if d.parkIfIdentityUnresolvable(ctx, task) {
return
}
trees, repo, err := d.taskWorkspace(ctx, task)
if err != nil {
d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err))
return
}
if d.ContainerPool == nil || d.Tasks == nil {
d.parkCapabilityUnavailable(ctx, task)
return
}
d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
workDir, ok := d.prepareWorkspace(ctx, task, trees, repo)
if !ok {
return
}
if !task.HasRepository() {
defer func() {
if err := trees.RemoveScratch(task.ID); err != nil {
d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err)
}
}()
}
// The workflow is pinned before the container starts, because the
// profile it names decides the container's image.
profile, ok := d.pinTaskProfile(ctx, task)
if !ok {
return
}
if profile.IsKit() {
d.runKitTask(ctx, task, repo, workDir, profile)
d.teardownStorage(ctx, task, repo, workDir)
d.cleanupTerminalTaskWorktree(ctx, task, trees)
return
}
ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image)
if !ok {
return
}
defer d.ContainerPool.Release(ctx, ctr)
defer revokeCredential()
// Run the whole task in archie-agent, which calls back to the daemon for
// store, forge and push operations.
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
runCtx, stopWatch := withContainerExit(limitCtx, ctr)
d.runViaAgent(runCtx, task, repo, profile, nil, credential)
stopWatch()
stopLimit()
d.teardownStorage(ctx, task, repo, workDir)
d.cleanupTerminalTaskWorktree(ctx, task, trees)
}
// teardownStorage runs after the workflow completes. The Docker backend is
// a no-op; future backends (temp volumes, NFS leases) use this hook.
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
if d.Storage == nil {
return
}
if err := d.Storage.Teardown(ctx, storage.TaskRef{
WorktreeDir: workDir,
Ecosystem: repo.Ecosystem,
PersistentStorage: repo.PersistentStorage,
Owner: task.Owner,
Repo: task.Repo,
}); err != nil {
d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err)
}
}
func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) {
if task.HasRepository() && d.ResolveRepository != nil {
resolved, err := d.Repository(ctx, task, config.OperationClone)
if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) {
// Checked before the run rather than at its commit step, so a
// missing permit parks in seconds instead of after the agent's work.
_, err = d.Repository(ctx, task, config.OperationPush)
}
return resolved.Trees, resolved.Repository, err
}
trees := d.treesFor(task)
repo, ok := d.repoFor(task)
if !ok && task.HasRepository() {
return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")}
}
return trees, repo, nil
}
// prepareWorkspace makes the directory the container binds as its
// workspace, which must exist before Acquire: a fresh clone of the task's
// repository, or an empty scratch directory for a task with none. It parks
// the task and reports false when it cannot.
func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) {
if !task.HasRepository() {
dir, err := trees.PrepareScratch(task.ID)
if err != nil {
d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err)
d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient)
return "", false
}
return dir, true
}
// The retry mode on the task decides whether to resume the pushed branch or
// reset onto base. Prepare clones a missing worktree either way.
target := worktree.Fresh
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork {
if task.Branch == "" {
// The action refuses this combination, so an empty branch here is a
// dispatch bug upstream and no retry can fix it.
d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID)
d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision)
return "", false
}
target = worktree.Target(task.Branch)
}
// Every task gets an independent full clone.
prepare := func() (string, string, error) {
return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
}
dir, branch, err := prepare()
if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) {
dir, branch, err = prepare()
}
if err != nil {
reason := "worktree prepare failed: " + err.Error()
if target != worktree.Fresh {
reason = "worktree resume failed: " + err.Error()
}
d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
return "", false
}
task.Branch = branch
if err := d.Store.Update(ctx, task); err != nil {
d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err)
}
return dir, true
}
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
// holds uncommitted work or its state cannot be read.
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
if d.Store == nil || trees == nil || task == nil || !task.HasRepository() {
return
}
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
if err != nil {
d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
return
}
if latest == nil {
return
}
// Settled statuses. Keep the worktree while a PR exists.
switch latest.Status {
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted:
if latest.PRNumber != 0 {
return
}
if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) {
return
}
if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil {
d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err)
}
}
}
// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has
// to survive cleanup because it may hold work no commit captured. A worktree
// that is not on disk at all has nothing to keep, so a task whose clone was
// already removed still cleans up whatever leftovers share its path.
func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool {
dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
if _, err := os.Stat(dir); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false
}
d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
return true
}
uncommitted, err := trees.HasUncommittedChanges(ctx, dir)
if err != nil {
d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
return true
}
if uncommitted {
d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir)
return true
}
return false
}
// openTaskLog opens the task's log sink for one process() call and returns
// its closer. Failure to open is logged, not fatal.
func (d *Daemon) openTaskLog(task *workflow.Task) func() {
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
}
return func() {
if err := d.TaskLogs.Close(task.ID); err != nil {
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
}
}
}
// acquireTaskContainer prepares and acquires the sandbox container for a
// task, reporting false when it could not -- having already parked the
// task and logged why. The caller releases the container it returns.
func (d *Daemon) acquireTaskContainer(
ctx context.Context,
task *workflow.Task,
repo config.Repo,
workDir string,
image string,
) (*container.Container, string, func(), bool) {
park := func(reason string, err error) {
d.Log.ErrorContext(ctx, reason, "err", err)
d.parkBlockedTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient, parkBlocker(err))
}
// Resolve the PR number from inputs before the brief is written.
if task.Workflow == "pr-review" {
task.PRNumber = task.EffectivePRNumber()
}
if err := writeTaskBrief(workDir, task); err != nil {
park("task.json write failed", err)
return nil, "", nil, false
}
// Fetch the pull request with the daemon's forge credential into workDir,
// so the container reads it without credentials.
if task.Workflow == "pr-review" {
if err := prefetchPRReview(ctx, d.forgeFor(ctx, task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil {
park("pr-review prefetch failed", err)
return nil, "", nil, false
}
}
// Guard: Storage may be nil if the daemon was wired incorrectly. In
// normal operation, Storage is always set when ContainerPool is set.
if d.Storage == nil {
d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container")
d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient)
return nil, "", nil, false
}
mounts, err := d.Storage.Setup(ctx, storage.TaskRef{
WorktreeDir: workDir,
Ecosystem: repo.Ecosystem,
PersistentStorage: repo.PersistentStorage,
Owner: task.Owner,
Repo: task.Repo,
})
if err != nil {
park("storage setup failed", err)
return nil, "", nil, false
}
credential, revokeCredential, err := d.runCredential(ctx, task)
if err != nil {
d.teardownStorage(ctx, task, repo, workDir)
park("run credential failed", err)
return nil, "", nil, false
}
model, closeModel, err := d.openModelEgress(ctx, task, credential)
if err != nil {
revokeCredential()
d.teardownStorage(ctx, task, repo, workDir)
park("model egress failed", err)
return nil, "", nil, false
}
revokeRun := revokeCredential
revokeCredential = func() { closeModel(); revokeRun() }
// The container lives in its own network namespace, so a loopback State
// Store target names the container itself. Give this task an address on
// the host bridge that forwards to it, closed with the task.
stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
if err != nil {
revokeCredential()
d.teardownStorage(ctx, task, repo, workDir)
park("state store relay failed", err)
return nil, "", nil, false
}
revokeCredential = release
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...))
if err != nil {
revokeCredential()
// Roll back the storage we just set up: the mounts were created
// for this task but no container will use them. Leaking them until
// a later TTL sweep is not acceptable on a backend that allocates
// real resources (volumes, NFS leases).
d.teardownStorage(ctx, task, repo, workDir)
park("container acquire failed", err)
return nil, "", nil, false
}
d.admitToRelay(ctx, relay, task, ctr)
return ctr, credential, revokeCredential, true
}
// writeTaskBrief writes task.json, the container's boot-time brief.
func writeTaskBrief(workDir string, task *workflow.Task) error {
return container.WriteTaskJSON(workDir, container.TaskPayload{
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
Labels: strings.Split(task.Labels, ","),
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
Inputs: task.Inputs,
})
}
// parkCapabilityUnavailable parks a task whose daemon-side capability is
// missing (container pool or task transport): both are environmental, so
// the park class is transient and the KindParked event carries the class
// for the dashboard. The reason distinguishes the two in logs and events.
func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) {
var reason string
if d.ContainerPool == nil {
reason = "managed agent container pool is unavailable; refusing to run this task on the host"
d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID,
"hint", "enable containers and make the archie-agent image available, then retry the task")
} else {
reason = "agent task transport is unavailable; refusing to run this task on the host"
d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID)
}
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
}
// parkRunningTask moves a running task to parked with class, using a
// bounded context that ignores cancellation. ErrStaleTransition is ignored;
// other errors are logged.
func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) {
d.parkBlockedTask(ctx, task, reason, class, "")
}
// parkBlockedTask is parkRunningTask recording the blocker the park site
// knows, so the dashboard can link to its fix.
func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) {
if d.Store == nil {
return
}
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil {
if !errors.Is(err, storecontract.ErrStaleTransition) {
d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err)
}
return
}
// Recorded only once the transition landed. An entry claiming the task
// parked would be a lie when another actor moved it out of running first
// -- the same race the parked event is withheld for.
d.recordPark(ctx, task.ID, reason)
d.emit(ctx, events.Event{
Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt,
Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow,
Detail: reason, Data: map[string]any{"park_class": class},
})
}
// recordPark writes a park reason to the attempt's log. Dropped when no sink
// is open.
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
d.TaskLogs.Write(ctx, taskID, logging.Entry{
Time: time.Now(),
Level: slog.LevelError.String(),
Message: "task parked: " + reason,
Fields: map[string]any{"component": "daemon", "task": taskID},
})
}
// runViaAgent hands a task to archie-agent and waits for its result. It parks
// the task only when the agent never answered or failed before recording an
// outcome.
//
// credential is the task's run credential; the agent presents it to push the
// task's branch.
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, harness *agentrun.HarnessSpec, credential string) {
cfg, err := d.taskConfig(ctx, task)
if err != nil {
d.parkRunningTask(ctx, task, err.Error(), taskstate.ParkTransient)
return
}
taskCfg := cfg.ForTask()
d.captureAttemptConfig(ctx, task, taskCfg)
var instructions workflow.Instructions
if d.TaskInstructions != nil {
instructions.Instance, instructions.Organisation, err = d.TaskInstructions.TaskInstructions(ctx, task.ID)
if err != nil {
d.parkRunningTask(ctx, task, "resolve the task's instructions: "+err.Error(), taskstate.ParkTransient)
return
}
}
req := taskrun.Request{
Instructions: instructions,
Task: task,
Repo: repo,
Cfg: taskCfg,
Providers: agentexec.ProvidersFromConfig(cfg.Providers),
MCPServers: cfg.Tools.MCPServers,
RunCredential: credential,
WorkflowDefinition: task.WorkflowDefinitionYAML,
Tools: d.toolAllowlist(ctx, task, profile.Tools),
Harness: harness,
}
req.Trace = map[string]string{}
otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace))
data, err := json.Marshal(req)
if err != nil {
d.Log.ErrorContext(ctx, "taskrun encode failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient)
return
}
reply, err := d.requestTaskRun(ctx, task.ID, data)
if err != nil {
if stoppingOnly(ctx) {
d.Log.InfoContext(ctx, "daemon stopping; task left running for the next daemon to re-queue", "task", task.ID)
return
}
err = runFailure(ctx, err)
d.Log.ErrorContext(ctx, "taskrun request failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "taskrun request failed: "+err.Error(), taskstate.ParkTransient)
return
}
var resp taskrun.Response
if err := json.Unmarshal(reply, &resp); err != nil {
d.Log.ErrorContext(ctx, "taskrun decode response failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient)
return
}
if resp.Error != "" {
d.Log.ErrorContext(ctx, "taskrun run failed", "task", task.ID, "err", resp.Error)
d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
return
}
// The agent applies the task's own outcome transition, so a status the
// task cannot reach from running means nothing landed: empty, unknown, or
// still running. Transient, because the cause is environmental -- a State
// Store that refused the transition -- and a retry can record one.
if !taskstate.CanTransition(taskstate.Running, resp.Status) {
reason := noOutcomeReason(resp.Status)
d.Log.ErrorContext(ctx, "taskrun run recorded no outcome", "task", task.ID, "status", resp.Status)
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
return
}
// Reached only past every failure return above, so this is always a
// fully decoded, error-free response -- a version is never recorded off
// a request that never reached an agent, or a response describing a
// parked/errored task.
if d.AgentStatus != nil && resp.AgentVersion != "" {
d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType)
}
d.Log.InfoContext(ctx, "taskrun complete", "task", task.ID, "status", resp.Status)
}
// noOutcomeReason names why a taskrun reply left the task without an outcome,
// quoting the status so the log line points at what the agent actually said.
func noOutcomeReason(status string) string {
if status == "" {
return "agent returned no status: the run could not record an outcome"
}
return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status)
}
// pinTaskProfile pins the task's workflow definition and resolves its agent
// profile, parking the task on failure.
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
if err := d.pinWorkflowDefinition(ctx, task); err != nil {
d.Log.ErrorContext(ctx, "pin workflow definition failed", "task", task.ID, "err", err)
d.parkBlockedTask(ctx, task, "pin workflow definition: "+err.Error(), pinParkClass(err), parkBlocker(err))
return config.AgentProfile{}, false
}
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
if err == nil {
// Every producer meets the declared interface here, including the
// ones that cannot read definitions when they enqueue (chat and
// dashboard spawns go through the Gateway, which never does).
err = iface.CheckInputs(task.Inputs)
}
if err == nil && iface.RepositoryMode() == workflowtask.RepositoryRequired && !task.HasRepository() {
// An optional workflow may hand off to one that requires a
// repository, because its task may carry one; a task that does not
// is parked here rather than run the target's steps in a scratch
// workspace.
err = fmt.Errorf("requires a repository and this task has none")
}
if err == nil {
var profile config.AgentProfile
if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil {
if err = validateProfileMeetsNeeds(profile, iface.Needs()); err == nil {
return profile, true
}
}
}
d.Log.ErrorContext(ctx, "agent profile unavailable", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "workflow "+task.Workflow+": "+err.Error(), taskstate.ParkNeedsDecision)
return config.AgentProfile{}, false
}
// validateProfileMeetsNeeds rejects a Kit profile whose harness cannot serve
// the captures the workflow needs. Image profiles always pass.
func validateProfileMeetsNeeds(profile config.AgentProfile, needs workflowtask.WorkflowNeeds) error {
if !profile.IsKit() || (!needs.Captures && len(profile.Tools) == 0) {
return nil
}
adapter, ok := agentexec.LookupHarnessAdapter(profile.Adapter)
if !ok || len(adapter.MCPConfig) == 0 {
return fmt.Errorf("profile adapter %q cannot register the required Archie MCP tools", profile.Adapter)
}
return nil
}
// pinWorkflowDefinition ensures the task carries a valid definition pin for
// the workflow it names and that its org has the workflow enabled. Every run
// passes through here, whatever started it, so this is the one enablement
// check for issue labels, handoffs, approvals, calls and playbooks alike.
func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error {
ctx, err := d.workflowContext(ctx, task)
if err != nil {
return err
}
if err := d.pinDefinition(ctx, task); err != nil {
return err
}
enablement, err := d.workflowEnablement(ctx)
if err != nil {
return pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow enablement: %w", err)}
}
if !enablement.Enabled(task.Org, task.Workflow) {
return blockedPark{taskstate.BlockerWorkflowDisabled, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow %s is disabled", task.Workflow)}}
}
if err := d.checkPackageTrigger(ctx, task); err != nil {
return pinFailure{taskstate.ParkNeedsDecision, err}
}
return nil
}
// pinDefinition keeps a valid pin and resolves a missing or mismatched one
// again from the active definitions.
func (d *Daemon) pinDefinition(ctx context.Context, task *workflow.Task) error {
pinned, ok, err := workflow.PinnedDefinitionID(task)
if err != nil {
return pinFailure{taskstate.ParkNeedsDecision, err}
}
if ok && pinned == task.Workflow {
return nil
}
d.runActionPlaybooks(ctx, task)
if d.WorkflowDefinitions == nil {
return d.pinWorkflowFromCollection(ctx, task, workflow.ShippedDefinitions(), 0)
}
collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx)
if err != nil {
return pinFailure{taskstate.ParkTransient, err}
}
return d.pinWorkflowFromCollection(ctx, task, collection, version)
}
// pinFailure carries the park class for a pin failure: operator-actionable
// for a bad pin or missing workflow, transient for a store error.
type pinFailure struct {
class taskstate.ParkClass
err error
}
func (e pinFailure) Error() string { return e.err.Error() }
func (e pinFailure) Unwrap() error { return e.err }
// pinParkClass reports the class a pin failure carries, defaulting to
// taskstate.ParkNeedsDecision for a cause that did not classify itself: the safe
// misread is "an operator should look at this", never "a requeue will fix it".
func pinParkClass(err error) taskstate.ParkClass {
if failure, ok := errors.AsType[pinFailure](err); ok {
return failure.class
}
return taskstate.ParkNeedsDecision
}
// blockedPark names what a park is missing alongside the failure.
type blockedPark struct {
blocker taskstate.Blocker
err error
}
func (e blockedPark) Error() string { return e.err.Error() }
func (e blockedPark) Unwrap() error { return e.err }
// parkBlocker reports the blocker err names, or empty.
func parkBlocker(err error) taskstate.Blocker {
if b, ok := errors.AsType[blockedPark](err); ok {
return b.blocker
}
if failure, ok := errors.AsType[*forge.AccessError](err); ok {
return accessBlocker(failure.Code)
}
return ""
}
// accessBlocker maps a repository access failure code onto what fixes it.
func accessBlocker(code string) taskstate.Blocker {
switch code {
case "operation_denied":
return taskstate.BlockerConnectionPermit
case "repository_not_configured", "connection_required", "connection_missing":
return taskstate.BlockerRepository
case "credential_unavailable", "account_unverified":
return taskstate.BlockerConnectionCredential
case "forge_unavailable":
return taskstate.BlockerForgeExtension
}
return ""
}
// runActionPlaybooks runs the action playbooks matching a task before it is
// pinned. Failures never fail the task; tasks already naming a workflow are
// skipped.
func (d *Daemon) runActionPlaybooks(ctx context.Context, task *workflow.Task) {
if d.Playbooks == nil || d.PlaybookLedger == nil || task.Workflow != "" {
return
}
if err := d.Playbooks.Run(ctx, d.PlaybookLedger, d.Log, playbookInput(task)); err != nil {
d.Log.WarnContext(ctx, "action playbook run failed", "task", task.ID, "err", err)
}
}
// resolveWorkflowID picks the definition to pin: a matching playbook, then
// the kind defaults. A task already naming a workflow keeps it.
func (d *Daemon) resolveWorkflowID(ctx context.Context, task *workflow.Task, available map[string]struct{}) (string, error) {
if d.Playbooks == nil || task.Workflow != "" {
return workflow.ResolveWorkflowID(task, available)
}
decision, matched := d.Playbooks.Dispatch(ctx, playbookInput(task))
if !matched {
return workflow.ResolveWorkflowID(task, available)
}
if _, defined := available[decision.Workflow]; !defined {
// Reported, not silently downgraded to the binding's choice: the
// operator bound this trigger to a named workflow, and running a
// different one under their rule is worse than parking the task.
return "", fmt.Errorf("playbook %q binds this trigger to workflow %q, which is not defined", decision.PlaybookID, decision.Workflow)
}
d.Log.InfoContext(ctx, "playbook selected workflow",
"task", task.ID, "playbook", decision.PlaybookID,
"playbook_version", decision.Version, "workflow", decision.Workflow)
return decision.Workflow, nil
}
// playbookInput builds the coordinator's dispatch input from a task row. The
// event surface is what the intake point actually knows about the originating
// issue -- the fields a `when` condition can read; it is deliberately not
// padded with values the daemon would have to invent.
func playbookInput(task *workflow.Task) playbook.DispatchInput {
labels := workintake.SplitLabels(task.Labels)
kind := string(workintake.KindForLabels(labels))
return playbook.DispatchInput{
Labels: labels,
Kind: kind,
TaskID: workintake.TaskEnvelope{Org: task.Org, Identity: task.Identity, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber}.IdempotencyKey(),
Event: map[string]any{
"kind": kind,
"labels": labels,
"owner": task.Owner,
"repo": task.Repo,
"number": task.IssueNumber,
"title": task.Title,
"body": task.Body,
"source": task.Source,
},
}
}
func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.Task, collection workflow.WorkflowDefinitionCollection, version int64) error {
available := make(map[string]struct{}, len(collection.Definitions))
for _, definition := range collection.Definitions {
available[definition.ID] = struct{}{}
}
id, err := d.resolveWorkflowID(ctx, task, available)
if err != nil {
return pinFailure{taskstate.ParkNeedsDecision, err}
}
definition, ok := collection.DefinitionByID(id)
if !ok {
return pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow definition %q disappeared", id)}
}
task.Workflow = id
task.WorkflowDefinitionVersion = version
task.WorkflowDefinitionYAML = definition.YAML
task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML)
if err := d.Store.Update(ctx, task); err != nil {
return pinFailure{taskstate.ParkTransient, fmt.Errorf("persist workflow definition pin: %w", err)}
}
return nil
}
const (
// defaultTaskRunReadyTimeout is how long requestTaskRun retries a
// nats.ErrNoResponders before giving up, when Daemon.TaskRunReadyTimeout
// is unset.
defaultTaskRunReadyTimeout = 20 * time.Second
// defaultTaskRunRetryBackoff is the delay between retry attempts, when
// Daemon.TaskRunRetryBackoff is unset.
defaultTaskRunRetryBackoff = 250 * time.Millisecond
)
func (d *Daemon) taskRunReadyTimeout() time.Duration {
if d.TaskRunReadyTimeout > 0 {
return d.TaskRunReadyTimeout
}
return defaultTaskRunReadyTimeout
}
func (d *Daemon) taskRunRetryBackoff() time.Duration {
if d.TaskRunRetryBackoff > 0 {
return d.TaskRunRetryBackoff
}
return defaultTaskRunRetryBackoff
}
// requestTaskRun publishes the taskrun request, retrying on
// nats.ErrNoResponders until the container subscribes. Other errors return
// at once.
func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) {
subject := agentnats.SubjectForTask(taskID)
deadline := time.Now().Add(d.taskRunReadyTimeout())
backoff := d.taskRunRetryBackoff()
for {
reply, err := d.Tasks.Request(ctx, subject, data)
if err == nil {
return reply, nil
}
if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) {
return nil, err
}
wait := min(backoff, time.Until(deadline))
if wait <= 0 {
return nil, err
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(wait):
}
}
}
// taskStateStoreEndpoint is the State Store address one task container is
// handed, the relay carrying it (nil when none is needed) and the release that
// ends that relay. The container lives
// in its own network namespace, so a loopback target names the container
// itself; behind a relay, the address is the host bridge's.
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) {
relay, err := d.openTaskStateStoreRelay(ctx)
if err != nil {
return "", nil, release, err
}
if relay == nil {
return d.ConnectedStateStore.URL, nil, release, nil
}
return relay.Addr(), relay, func() { relay.Close(); release() }, nil
}
// admitToRelay lets ctr, and only ctr, through relay. A container whose
// addresses cannot be read is refused rather than let anyone through.
func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) {
if relay == nil {
return
}
addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID)
if err != nil {
d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err)
}
relay.Admit(addrs)
}
// openTaskStateStoreRelay starts the relay that carries one task container's
// State Store traffic to the daemon's target. The target stays bound where the
// operator put it; the container is handed an address on the host bridge, the
// same way it is handed NATS's. Nil when the target already reaches the
// containers, or when no bridge address is known.
func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) {
addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL)
if !loopback {
return nil, nil
}
gateway := ""
if d.ContainerPool != nil {
gateway = d.ContainerPool.HostGateway()
}
if gateway == "" {
d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it",
"target", d.ConnectedStateStore.URL)
return nil, nil
}
return openStateStoreRelay(ctx, gateway, addr)
}
// containerEnv is the environment a task container starts with. stateStoreURL
// is the address the container can reach the State Store by, which is the
// relay's rather than the daemon's own target.
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string {
var env []string
// The endpoint the daemon's own client connected with at startup, not
// the live config: a reloaded [nats] section must not point new
// containers at a server the daemon is not publishing on.
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
if token := d.ConnectedNATS.ContainerToken(); token != "" {
env = append(env, "NATS_TOKEN="+token)
}
// Pass the container-reachable endpoint with this task's scoped token,
// never the daemon's own.
if stateStoreURL != "" {
env = append(env, "STATE_STORE_URL="+stateStoreURL)
if len(d.ConnectedStateStore.CA) > 0 {
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
}
if stateStoreToken != "" {
env = append(env, "STATE_STORE_TOKEN="+stateStoreToken)
}
}
// The agent runs as root in the container, so its commits land owned by
// UID 0 on the host. The daemon's UID/GID let the agent hand the worktree
// back before the daemon reads it to push.
env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()))
env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid()))
return env
}
// runCredential issues the task's run credential, or "" when no State Store
// is configured. It fails when no issuer is wired.
func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) {
if d.ConnectedStateStore.URL == "" {
return "", func() {}, nil
}
if d.RunCredentials == nil {
return "", nil, fmt.Errorf("state store is configured but no run credential issuer is wired")
}
return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std())
}
func (d *Daemon) configFor(task *workflow.Task) config.Config {
cfg := d.Cfg.Get()
if !d.rootTask(task) {
cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{}
cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil
cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil
}
return cfg
}
// captureAttemptConfig records the attempt's effective configuration as an
// event. Failures are logged.
func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) {
if d.Store == nil {
return
}
document, err := json.Marshal(cfg)
if err != nil {
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
return
}
event := events.Event{
At: time.Now().UTC(),
Kind: events.KindConfigCaptured,
TaskID: task.ID,
Repo: task.Owner + "/" + task.Repo,
Issue: task.IssueNumber,
Workflow: task.Workflow,
Attempt: task.Attempt,
Data: map[string]any{
"schema": events.ConfigCapturedSchema,
// Raw so the document is embedded once, as it was encoded, rather
// than as a string a reader would have to decode again.
"document": json.RawMessage(document),
},
}
id, err := d.Store.InsertEvent(ctx, event)
if err != nil {
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
return
}
// The assigned ID tells the event sink to broadcast without inserting a
// duplicate row.
event.ID = id
d.emit(ctx, event)
}
// identityMayAct reports whether a task's identity may act, and the park
// class if not. Empty is the root identity. An unknown or inactive identity
// never falls back to the root forge.
func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) {
id := d.publisherIdentity(name)
if d.IdentityRepository != nil {
value, err := d.IdentityRepository.Get(ctx, id)
if err != nil {
return false, "execution identity unavailable: " + err.Error(), taskstate.ParkTransient
}
if value.Lifecycle != identity.LifecycleActive {
return false, "identity may not act", taskstate.ParkNeedsDecision
}
return true, "", ""
}
if name == "" || name == string(d.RootIdentityID) {
return true, "", ""
}
return false, "identity " + name + " no longer resolves", taskstate.ParkTransient
}
// parkIfIdentityUnresolvable parks task when its non-empty identity cannot be
// acted on, reporting whether it did. An empty identity is the root
// single-identity path and never parks here.
func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool {
ok, reason, class := d.identityMayAct(ctx, task.Identity)
if ok {
return false
}
d.parkRunningTask(ctx, task, reason, class)
return true
}
func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool {
if d.IdentityRepository == nil {
return true
}
if id == "" {
d.Log.ErrorContext(ctx, "identity lifecycle check failed", "err", "identity ID is empty")
return false
}
value, err := d.IdentityRepository.Get(ctx, id)
if err != nil {
d.Log.ErrorContext(ctx, "identity lifecycle check failed", "identity", id, "err", err)
return false
}
return value.Lifecycle == identity.LifecycleActive
}
// forgeFor returns the forge client that owns task: its organisation
// connection, else the root d.Forge for the root alone. This is the safety
// boundary that keeps one identity's forge token from being used against
// another identity's repos.
func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge {
if d.ResolveRepository != nil && task.HasRepository() {
resolved, err := d.Repository(ctx, task, config.OperationClone)
if err != nil {
return forge.NewNoop(d.Log)
}
return resolved.Forge
}
if !d.rootTask(task) {
return forge.NewNoop(d.Log)
}
return d.Forge
}
// treesFor returns the worktree manager that owns task, mirroring forgeFor.
func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager {
if d.ResolveRepository != nil && task.HasRepository() {
// Cleanup needs only the org's path, not a still-valid external credential.
return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())}
}
if !d.rootTask(task) {
return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))}
}
return d.Trees
}
func (d *Daemon) rootTask(task *workflow.Task) bool {
return task.Identity == "" || identity.IdentityID(task.Identity) == d.RootIdentityID
}
// repoFor resolves a root task's repo config from Cfg.Repos.
func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) {
if !d.rootTask(t) {
return config.Repo{}, false
}
for _, r := range d.Cfg.Get().Repos {
if r.Owner == t.Owner && r.Name == t.Repo {
return r, true
}
}
return config.Repo{}, false
}
// allowConcurrentForTask reports whether the task's repo allows concurrent
// tasks. Unknown repos do not.
func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
if !task.HasRepository() {
return true
}
repo, ok := d.repoFor(task)
return ok && repo.AllowConcurrent
}
// StalestLoop reports the run loop that least recently began a pass and when,
// or the zero time when no loop has started yet. The systemd watchdog beats
// only while this is fresh, so every loop Run starts must mark its passes.
func (d *Daemon) StalestLoop() (string, time.Time) {
var loop string
var oldest int64
d.loopPasses.Range(func(k, v any) bool {
name, okName := k.(string)
if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) {
loop, oldest = name, ns
}
return true
})
if oldest == 0 {
return "", time.Time{}
}
return loop, time.Unix(0, oldest).UTC()
}
// markPass stamps the start of one loop's pass, so a hung pass shows as stale.
func (d *Daemon) markPass(loop string) {
d.loopPasses.Store(loop, time.Now().UnixNano())
}
// errContainerExited is the cancellation cause when a task's agent container
// stops before answering its taskrun request.
var errContainerExited = errors.New("agent container exited before the task finished")
// runFailure names why a taskrun request failed: the container's exit or the
// time limit when either cancelled it, else the request's own error.
func runFailure(ctx context.Context, err error) error {
if cause := context.Cause(ctx); errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit) {
return cause
}
return err
}
// exitWatcher is a running agent container, seen only for how it ends.
type exitWatcher interface {
Exited() <-chan struct{}
ExitDetail() string
}
// withContainerExit derives a context cancelled when exited closes. A dead
// container never answers its taskrun request, so without this the request
// blocks forever and the task row stays running.
func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) {
runCtx, cancel := context.WithCancelCause(ctx)
go func() {
select {
case <-ctr.Exited():
if detail := ctr.ExitDetail(); detail != "" {
cancel(fmt.Errorf("%w: %s", errContainerExited, detail))
return
}
cancel(errContainerExited)
case <-runCtx.Done():
}
}()
return runCtx, func() { cancel(nil) }
}
// errTaskTimeLimit is the cancellation cause when a task outruns its overall
// time limit (the execution settings' max task runtime).
var errTaskTimeLimit = errors.New("task exceeded its time limit")
// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies
// no bound.
func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) {
if limit <= 0 {
return ctx, func() {}
}
return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit))
}
// RemoveWorktree deletes a task's clone from the trees of the identity that
// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold
// uncaptured work, and the operator's archive is the decision to discard it.
func (d *Daemon) RemoveWorktree(task *workflow.Task) error {
trees := d.treesFor(task)
if trees == nil {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber)
}
// cleanupWorktree removes a task's clone, first handing back any root-owned
// files a killed container left in it.
func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error {
err := trees.Cleanup(owner, repo, issue)
if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) {
return trees.Cleanup(owner, repo, issue)
}
return err
}
// reownDenied repairs dir when err says the daemon was refused access to it,
// and reports whether the caller should retry. A SIGKILLed task container
// leaves root-owned files that only a container can chown back.
func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool {
if err == nil || !errors.Is(err, fs.ErrPermission) || d.ContainerPool == nil {
return false
}
if rerr := d.ContainerPool.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil {
d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr)
return false
}
return true
}
// InFlight counts the same submitted work that graceful shutdown waits for,
// including tasks waiting for a repository or concurrency slot.
func (d *Daemon) InFlight() int {
dispatcher := d.taskDispatcher()
dispatcher.mu.Lock()
defer dispatcher.mu.Unlock()
return dispatcher.pending
}