// Package daemon is archied's resident loop: poll GitHub for labelled // issues, enqueue them, and process tasks through their routed workflows. // State lives in the store; the daemon is restartable at any point. package daemon import ( "context" "encoding/json" "errors" "fmt" "io/fs" "log/slog" "maps" "os" "path/filepath" "strings" "sync" "time" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/domain/access" "github.com/samcharles93/archie-core/internal/domain/agentrun" "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/eda/playbook" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/workflow" workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/forge" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/natsrpc" "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/taskrun" "github.com/samcharles93/archie-core/internal/taskstate" "github.com/samcharles93/archie-core/internal/worktree" ) // TaskBus publishes discovered work, collects it back, and asks a worker to // run a task. type TaskBus interface { PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error Fetch(ctx context.Context) (eventbus.Message, error) Request(ctx context.Context, subject string, payload []byte) ([]byte, error) } // RunCredentialIssuer issues the one credential a task's container holds: it // authorizes the task's own State Store calls and the push of its branch. // It lapses at the task's time limit plus a margin if never revoked. type RunCredentialIssuer interface { Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error) } // NATSEndpoint is the broker address and credential archied connected with at // startup. It is runtime state rather than configuration: embedded mode // generates both values, and external secret references are resolved once. type NATSEndpoint struct { URL string Token string // Scoped is true on the embedded broker, where a container logs in with // its run credential and reaches only its own task's subjects. Scoped bool } // ContainerToken is the broker token a container is handed: none on a // scoped broker. An external broker cannot scope a container, so it still // gets the broker's token (reported as a health issue by archied). func (e NATSEndpoint) ContainerToken() string { if e.Scoped { return "" } return e.Token } // StateStoreEndpoint is the State Store target and the CA its certificate // chains to, both passed to agent containers. type StateStoreEndpoint struct { URL string CA []byte } type Daemon struct { ResolveRepository func(context.Context, *workflow.Task, string) (RepositoryAccess, error) // Cfg publishes the running configuration. Read through Cfg() so a // reload swaps the published snapshot atomically. See config.Holder. Cfg *config.Holder // ConnectedNATS is the endpoint the daemon connected with at startup; // container env uses it, never live config. ConnectedNATS NATSEndpoint // ConnectedStateStore is the State Store target the daemon dialed at // startup, passed to containers. ConnectedStateStore StateStoreEndpoint Store storecontract.TaskStore // Mappings persists payload field mappings. // Used by the binding dispatch loop to resolve capture bodies against // the mapping a binding names. Optional: nil disables the binding // dispatch loop (legacy behaviour). Mappings storecontract.MappingStore // Bindings persists playbook bindings. // Optional: nil disables the binding dispatch loop (legacy behaviour). Bindings storecontract.BindingStore // BindingDispatcher lists undispatched captures, finds armed bindings and // records dispatches. Nil disables binding dispatch. BindingDispatcher storecontract.BindingDispatcher // MappingMatches counts each event a mapping resolved at dispatch. // Optional: nil leaves match counts unrecorded. MappingMatches storecontract.MappingMatchRecorder // BindingTaskCreator enqueues a task triggered by a binding. Split // off TaskLifecycle so the lifecycle surface stays narrow and the // dispatch loop does not acquire the full task-creation contract. // Optional: nil disables the dispatch loop. BindingTaskCreator storecontract.BindingTaskCreator // Access evaluates the policy chain at dispatch, the second of the two // Authorizer call sites: the workflow's identity may `run` the workflow in // its workspace. Optional: nil dispatches without the chain, which is the // behaviour of an install that has not built it. Access access.Authorizer // Principals assembles the dispatch principal for the workflow's // identity. Wired with Access or not at all. Principals access.PrincipalSource // Denials records dispatch refusals. Optional: nil skips the record. Denials access.DenialRecorder Forge forge.Forge Trees *worktree.Manager Bus *events.Bus Log *slog.Logger // Tasks is the NATS task bus. Always set in production. Tasks TaskBus // reactionConsumer is built lazily by drainReactions and kept for its // lifetime drop counter. reactionConsumer *reactionConsumer // lastPolled remembers each repo's eligible issues from its previous // complete poll, under the dispatch rule that produced them, so only an // issue that was eligible and dropped out reads as withdrawn. lastPolledMu sync.Mutex lastPolled map[string]polledIssues // reactionPublisher is built lazily by scanPRReviews; an indirection // only so tests can swap the delivery without the bus. reactionPublisher ReactionPublisher // Reactions is the review-reaction stream, drained into remediate runs each // cycle. Nil disables it. Reactions ReactionSource // RunCredentials issues each task's run credential (see runCredential). // Required whenever ConnectedStateStore.URL is set: the task is parked // rather than handed the daemon's own administrative token. RunCredentials RunCredentialIssuer // KitLauncher starts tasks whose agent profile is a Kit. Nil parks them. KitLauncher KitLauncher // ModelEgress gives native task containers model access without provider // keys. Nil parks a task whose providers need a key. ModelEgress ModelEgress // TaskModels answers the model aliases a task's org resolves against. // Nil runs every task on the instance aliases. TaskModels TaskModels // TaskInstructions answers the instance and organisation instructions a // task's agent runs follow. Nil runs without them. TaskInstructions TaskInstructions // TaskRunReadyTimeout bounds how long runViaAgent retries a taskrun request // while the new container has not subscribed yet. Zero uses // defaultTaskRunReadyTimeout. TaskRunReadyTimeout time.Duration // TaskRunRetryBackoff is the delay between retry attempts within // TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff. TaskRunRetryBackoff time.Duration // AgentStatus records agent versions for update verification. Nil disables // it. AgentStatus *AgentStatus // ContainerPool manages Docker container lifecycle. Nil means autonomous // execution is unavailable and process parks tasks; every runnable task gets // a fresh container. ContainerPool *container.Pool // dispatcher bounds task execution globally across poll passes. Built once, // on first use, so ResizeTaskDispatcher can apply a live // containers.max_concurrency change without a restart; drainNATS submits // each pass through it and leaves its tasks running. dispatcherOnce sync.Once dispatcher *taskDispatcher // Storage is the pluggable storage backend for container mounts. // A runnable task requires it; acquireTaskContainer parks on nil. Storage storage.Backend // IssueSources resolves, each poll cycle, every organisation connection // to poll and the identity whose permit reaches it. Nil polls nothing. IssueSources func(context.Context) []IssueSource // OrgPrincipals is one acting principal per organisation, system org // first. Nil dispatches the root identity's org alone. OrgPrincipals func(context.Context) []access.Principal // IdentityRepository is the durable lifecycle authority. Polling checks it // every cycle so suspend/reactivate commands do not require a restart. IdentityRepository identity.Repository AgentAssignments org.AgentAssignments RootIdentityID identity.IdentityID // PackageAuthorities bounds the tools a package workflow's run is offered. PackageAuthorities PackageAuthorities // Playbooks are the loaded EDA playbooks, consulted before kind/label // bindings when pinning a workflow. Nil means none. Playbooks interface { Dispatch(context.Context, playbook.DispatchInput) (playbook.Decision, bool) Run(context.Context, storecontract.PlaybookDispatcher, *slog.Logger, playbook.DispatchInput) error } // PlaybookLedger is the at-most-once gate action playbooks run through. // Nil means action playbooks do not run: they never fire unguarded. PlaybookLedger storecontract.PlaybookDispatcher // WorkflowDefinitions supplies the active database definitions. A task is // pinned once before dispatch; retries reuse the task's stored YAML. WorkflowDefinitions interface { WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error) } // WithdrawTask declines a task whose issue a complete poll no longer // returns. Nil leaves withdrawn issues' work running. WithdrawTask func(ctx context.Context, taskID int64, reason string) error // WorkflowEnablement supplies which workflows each org has disabled. A // binding targeting a workflow its org disabled does not dispatch. Nil // means every workflow is enabled. WorkflowEnablement interface { WorkflowEnablement(context.Context) (workflowtask.WorkflowEnablement, error) } // TaskLogs persists each task's log output and mirrors it to the dashboard. // Nil disables task logging. TaskLogs *logging.TaskRegistry // running holds a cancel function for every task currently executing, // so /stop can reach work already in flight. Its zero value is ready // to use. running runningTasks // lastPollAt maps each poll loop (an identity ID, or "" for the root // loop) to when its latest pass began, as Unix nanoseconds. LastPollAt // reports the oldest, so one wedged loop is not hidden by a healthy one. lastPollAt sync.Map } // IssueSource is one forge connection's repositories as the identity // permitted to read them sees them. type IssueSource struct { Identity identity.IdentityID Forge forge.Forge // BotUser is the connection's external account, the assignee polled for. BotUser string Repos []config.Repo } func (d *Daemon) issueSources(ctx context.Context) []IssueSource { if d.IssueSources == nil { return nil } return d.IssueSources(ctx) } // sourceTag is the identity a source's tasks carry: empty for the root. func (d *Daemon) sourceTag(source IssueSource) string { if source.Identity == d.RootIdentityID { return "" } return string(source.Identity) } // emit publishes an observability event; safe on a nil bus. func (d *Daemon) emit(ctx context.Context, e events.Event) { if d.Bus != nil { d.Bus.Publish(ctx, e) } } // Startup runs crash recovery and access verification once. func (d *Daemon) Startup(ctx context.Context) error { if err := d.cleanupExpiredStorage(ctx); err != nil { d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err) } n, err := d.Store.RecoverStale(ctx) if err != nil { return err } if n > 0 { d.Log.InfoContext(ctx, "re-queued tasks left running by a previous daemon", "count", n) } d.sweepAccess(ctx) return nil } // sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and // the run loop, and a forge client carries no request deadline of its own, so // an unreachable host would otherwise hold boot open indefinitely. const sweepTimeout = 2 * time.Minute // sweepAccess accepts pending invitations and verifies push access through // every organisation connection, each in its own goroutine. func (d *Daemon) sweepAccess(ctx context.Context) { sweep := func(source IssueSource) { log := d.Log.With("identity", source.Identity) ctx, cancel := context.WithTimeout(ctx, sweepTimeout) defer cancel() if err := source.Forge.AcceptInvitations(ctx); err != nil { log.WarnContext(ctx, "invitation sweep failed", "err", err) } for _, r := range source.Repos { if err := source.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil { // A sweep cut short by its own context is not a repo failure: name // the budget when it expired, stay quiet on a shutdown. switch { case errors.Is(ctx.Err(), context.DeadlineExceeded): // Every later call would return the same error, so say it once. log.WarnContext(ctx, "push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err) return case ctx.Err() != nil: return } log.WarnContext(ctx, "repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err) } } } var wg sync.WaitGroup for _, source := range d.issueSources(ctx) { wg.Go(func() { sweep(source) }) } wg.Wait() } // Run polls until the context ends. func (d *Daemon) Run(ctx context.Context) error { // A pass no longer waits for the tasks it submitted, so the run does, once // the ctx is cancelled. defer d.WaitForTasks() d.watchCallWaits(ctx) if d.Tasks != nil { drained := make(chan struct{}) defer func() { <-drained }() go func() { defer close(drained) d.drainLoop(ctx) }() } interval := d.Cfg.Get().PollInterval.Std() ticker := time.NewTicker(interval) defer ticker.Stop() for { d.Cycle(ctx) select { case <-ctx.Done(): return ctx.Err() case <-ticker.C: // PollInterval may have been reloaded; reset the ticker so a // change takes effect on the next cycle. if iv := d.Cfg.Get().PollInterval.Std(); iv != interval { interval = iv ticker.Reset(iv) } } } } // drainLoop is the only task drainer while Run is active. It runs apart from // the poll tick so a queued task starts within about a second regardless of // poll_interval or forge latency. Submit never blocks, so a full dispatcher // just leaves tasks waiting on slots and the loop stays paced by the ticker. func (d *Daemon) drainLoop(ctx context.Context) { ticker := time.NewTicker(time.Second) defer ticker.Stop() for { d.drainNATS(ctx) select { case <-ctx.Done(): return case <-ticker.C: } } } // pollIssuesWithConfig returns the repo's issues the dispatch rule makes // work, using the given forge client and dispatch config. complete is false when any part of the poll failed or was skipped, so // the result must not be read as the whole eligible set. func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) { // Webhook intake hears about issues as they change; polling as well would // spend forge API quota to find the same work. The receiver runs from the // root forge settings, so their mode decides for every identity. if d.Cfg.Get().Forge.Intake == config.ForgeIntakeWebhook { return nil, false } labelled := func() ([]forge.Issue, bool) { // Empty-label rule shared with workintake.MatchesDispatch: a label // trigger needs a non-empty label or it would match every open issue. if cfg.Label == "" { d.Log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName()) return nil, false } issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label) if err != nil { d.Log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err) return nil, false } return issues, true } assigned := func() ([]forge.Issue, bool) { if cfg.BotUser == "" { d.Log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName()) return nil, false } issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser) if err != nil { d.Log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err) return nil, false } return issues, true } switch cfg.Dispatch.Trigger { case "label": return labelled() case "either": byAssignee, assignedOK := assigned() byLabel, labelledOK := labelled() seen := map[int]bool{} for _, is := range byAssignee { seen[is.Number] = true } issues = byAssignee for _, is := range byLabel { if !seen[is.Number] { issues = append(issues, is) } } return issues, assignedOK && labelledOK default: // "assignee" (default) return assigned() } } // RunOnce runs one cycle and drains until no task is in flight. A run waiting // on a callee stays in flight while the callee sits queued, so the queue is // drained again until every task, callees included, has returned. func (d *Daemon) RunOnce(ctx context.Context) { defer d.WaitForTasks() d.watchCallWaits(ctx) // One pass has no scheduler, so it runs the maintenance jobs itself. for _, job := range d.BuiltinJobs() { if err := job.Run(ctx); err != nil { d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err) } } d.Cycle(ctx) if d.Tasks == nil { return } d.drainNATS(ctx) ticker := time.NewTicker(time.Second) defer ticker.Stop() for d.taskDispatcher().inFlight() > 0 { select { case <-ctx.Done(): return case <-ticker.C: d.drainNATS(ctx) } } } // Cycle is one poll pass: turn captured events and review reactions into // tasks; maintenance runs as built-in schedules. Draining queued tasks is the caller's job // (drainLoop in Run, drainNATS in RunOnce) so there is one drainer. func (d *Daemon) Cycle(ctx context.Context) { d.dispatchBindings(ctx) d.drainReactions(ctx) } func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error { ttl := d.Cfg.Get().Containers.VolumeTTL.Std() if ttl <= 0 || d.Storage == nil { return nil } n, err := d.Storage.CleanupExpired(ctx, ttl) if n > 0 { d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n) } if err != nil { return fmt.Errorf("persistent volume cleanup: %w", err) } return nil } // bindingDispatchBatchLimit caps how many undispatched captures a single // dispatchBindings pass will fetch from the store. The cap is defensive // (the captures table is also pruned by retention/maxEvents) and bounds a // single cycle's worst-case latency on a backlog. const bindingDispatchBatchLimit = 100 // dispatchBindings offers each identified capture to every armed binding on // its source. Nil Bindings, BindingDispatcher or BindingTaskCreator disables // it. func (d *Daemon) dispatchBindings(ctx context.Context) { if d.Bindings == nil || d.BindingDispatcher == nil || d.BindingTaskCreator == nil { return } if d.Principals == nil { d.dispatchOrgBindings(ctx) return } var principals []access.Principal if d.OrgPrincipals != nil { principals = d.OrgPrincipals(ctx) } else { principal, err := d.Principals.PrincipalFor(ctx, d.RootIdentityID) if err != nil { d.Log.WarnContext(ctx, "binding dispatch: principal unavailable", "identity", d.RootIdentityID, "err", err) return } principals = []access.Principal{principal} } seen := make(map[org.OrgID]bool) for _, principal := range principals { if seen[principal.Org] { continue } seen[principal.Org] = true d.dispatchOrgBindings(org.WithOrg(access.WithPrincipal(ctx, principal), principal.Org)) } } func (d *Daemon) dispatchOrgBindings(ctx context.Context) { // Gather the armed bindings whose workflow is enabled first, so the // oldest-first captures query is not held by captures that only a binding // on a disabled workflow would take. bindings, err := d.Bindings.ListBindings(ctx) if err != nil { d.Log.WarnContext(ctx, "list bindings failed", "error", err) return } enablement, err := d.workflowEnablement(ctx) if err != nil { d.Log.WarnContext(ctx, "binding dispatch: workflow enablement unavailable", "error", err) return } enabled := make([]string, 0, len(bindings)) for _, b := range bindings { if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) { enabled = append(enabled, b.ID) } } if len(enabled) == 0 { return } captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit) if err != nil { d.Log.WarnContext(ctx, "list undispatched captures failed", "error", err) return } workflows, err := d.activeWorkflows(ctx) if err != nil { d.Log.WarnContext(ctx, "binding dispatch: workflow definitions unavailable", "error", err) return } for _, c := range captures { d.dispatchCapture(ctx, c, workflows, enablement) } } // dispatchCapture offers one capture to every armed binding on its source. A // disabled workflow's binding is skipped like an unarmed one, so the capture // waits rather than recording a failure every cycle. func (d *Daemon) dispatchCapture(ctx context.Context, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection, enablement workflowtask.WorkflowEnablement) { armed, err := d.BindingDispatcher.ArmedBindingsForSource(ctx, c.Source) if err != nil { d.Log.WarnContext(ctx, "armed bindings lookup", "source", c.Source, "error", err) return } for _, b := range armed { if b.OrgID == org.OrgFromContext(ctx) && b.Matcher.Matches(c.Source, c.Dispatchable()) && enablement.Enabled(b.OrgID, b.Workflow) { d.dispatchOneBinding(ctx, b, c, workflows) } } } // bindingMapping returns b's mapping, or nil after logging why it is // unavailable. A mapping that no longer exists is a terminal outcome. func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent) *mapping.Mapping { if d.Mappings == nil { d.Log.WarnContext(ctx, "binding dispatch: mapping store unavailable", "binding", b.ID) return nil } m, err := d.Mappings.GetMapping(ctx, b.MappingID) if err != nil { d.Log.WarnContext(ctx, "binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err) return nil } if m == nil { d.recordDispatchFailure(ctx, b, c, fmt.Sprintf("mapping %q does not exist", b.MappingID), nil) } return m } // dispatchOneBinding offers one capture to one binding: the mapping must // match the capture's event type and resolve required fields, the filter // must admit the values, and the workflow must accept the inputs. The // ledger claim and the task commit together, so each binding fires exactly // once per delivery. func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) { m := d.bindingMapping(ctx, b, c) if m == nil || m.EventTypeID != c.EventType { return } values, failures := mapping.Resolve(m.Fields, []byte(c.Body)) if hasBlockingFailure(m.Fields, failures) { d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures}) return } if d.MappingMatches != nil { if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil { d.Log.WarnContext(ctx, "binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err) } } filter, err := binding.CompileFilter(b.Filter, m.Fields) if err != nil { d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()}) return } if admitted, _ := filter.Admits(values); !admitted { d.recordEvaluated(ctx, b, c, "filtered") return } target, reason, ok := d.resolveBindingTarget(b, values, workflows) if !ok { d.recordDispatchFailure(ctx, b, c, reason, nil) return } // The chain decides dispatch: the binding's principal may `run` this // workflow in its workspace, and the event's signature result and address // travel with the request. A denial is terminal; an unavailable principal // or policy chain leaves the capture listed for the next cycle. if !d.authorizeDispatch(ctx, b, c, target) { return } d.claimAndEnqueue(ctx, b, c, target, values) } // claimAndEnqueue asks the State Store to claim the dispatch and enqueue its // task as one unit. A failure leaves the delivery undispatched, so the next // cycle offers it again; a delivery already dispatched is refused. func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) { title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source) body := renderBindingBody(values, c) if target.declared { body = fmt.Sprintf("Started by binding %q from capture %q; the workflow's inputs travel with the task.", b.Name, c.ID) } taskIdentity := "" if principal, ok := access.PrincipalFromContext(ctx); ok && (principal.IdentityID != d.RootIdentityID || principal.Org != org.DefaultOrgID) { taskIdentity = string(principal.IdentityID) } task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs) if err != nil { if !errors.Is(err, storecontract.ErrAlreadyDispatched) { d.Log.WarnContext(ctx, "binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err) } return } d.markBindingStart(ctx, task.ID, b, c) } // authorizeDispatch checks the binding's principal, its org's dispatching // identity, may start the workflow. A denial is recorded and reported as a // binding_dispatch_failure. The run itself acts as the workflow's declared // identity, which adoptWorkflowIdentity checks when the task is processed. func (d *Daemon) authorizeDispatch(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget) bool { if d.Access == nil || d.Principals == nil { return true } principal, ok := access.PrincipalFromContext(ctx) if !ok { d.Log.WarnContext(ctx, "binding dispatch: no principal; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID) return false } resource := access.Resource{ Kind: access.KindWorkflow, ID: b.Workflow, Org: principal.Org, Workspace: b.WorkspaceID, } contextValue := access.Context{Signature: signatureOf(c), Addr: c.RemoteAddr} decision := d.Access.Authorize(principal, access.ActionRun, resource, contextValue) if decision.Allowed { return true } if errors.Is(decision.Err, access.ErrChainUnavailable) { // Nothing was evaluated. Recording it would mark the pair evaluated for // good, so a State Store blip would drop the capture instead of leaving // it for the next cycle. d.Log.WarnContext(ctx, "binding dispatch: access chain unavailable; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID, "err", decision.Err) return false } if d.Denials != nil { denial := access.Denial{ Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun, Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level, Policies: decision.Policies, } if err := d.Denials.RecordDenial(ctx, denial); err != nil { d.Log.WarnContext(ctx, "binding dispatch: record denial", "binding", b.ID, "error", err) } } d.recordDispatchFailure(ctx, b, c, "access denied", map[string]any{ "level": string(decision.Level), "policies": decision.Policies, }) return false } // signatureOf names the event's signature result as the dispatch context // carries it: a signed event and an approved unsigned event are the two // dispatchable shapes (storecontract.CapturedEvent.Dispatchable). func signatureOf(c storecontract.CapturedEvent) string { switch { case c.Authenticated: return "valid" case c.Unsigned: return "unsigned" } return "unverified" } // markBindingStart records on a task's timeline the binding and captured // event that started it. func (d *Daemon) markBindingStart(ctx context.Context, taskID int64, b binding.Binding, c storecontract.CapturedEvent) { if _, err := d.Store.InsertEvent(ctx, events.Event{ TaskID: taskID, Kind: events.KindBindingStarted, Detail: fmt.Sprintf("started by binding %q from source %q", b.Name, c.Source), Data: map[string]any{ "binding_id": b.ID, "binding_name": b.Name, "binding_version": b.Version, "capture_id": c.ID, "source": c.Source, "unsigned": c.Unsigned, }, }); err != nil { d.Log.WarnContext(ctx, "binding dispatch: start marker", "task", taskID, "error", err) } } // recordDispatchFailure records why a binding did not dispatch a capture and // marks the pair evaluated, so the capture is not offered to it again. func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) { d.recordEvaluated(ctx, b, c, reason) data := map[string]any{ "binding_id": b.ID, "binding_version": b.Version, "capture_id": c.ID, } maps.Copy(data, extra) _, _ = d.Store.InsertEvent(ctx, events.Event{ Kind: "binding_dispatch_failure", Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason), Data: data, }) } // recordEvaluated writes the ledger row for an evaluation that started no // task. Only terminal outcomes call it; transient ones leave the capture // listed for the next cycle. func (d *Daemon) recordEvaluated(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string) { err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, reason) if err != nil && !errors.Is(err, storecontract.ErrAlreadyDispatched) { d.Log.WarnContext(ctx, "binding dispatch: record evaluation", "binding", b.ID, "capture", c.ID, "error", err) } } // hasBlockingFailure reports whether any failure is on a required field. func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool { if len(failures) == 0 { return false } required := make(map[string]bool, len(fields)) for _, f := range fields { if f.Required { required[f.Name] = true } } for _, fail := range failures { if required[fail.FieldName] { return true } } return false } // renderBindingBody builds the task body from the values a mapping // resolved, one "key=value" pair per line. Empty maps still produce // a body that records the source capture id so the workflow has // provenance to hand back to the operator. func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string { if len(values) == 0 { return fmt.Sprintf("(no fields resolved from capture %q)", c.ID) } parts := make([]string, 0, len(values)) for k, v := range values { parts = append(parts, fmt.Sprintf("%s=%v", k, v)) } return strings.Join(parts, "\n") } // resolveBindingRepo returns the binding's pinned owner/repo, or the only // configured repo. With zero or several repos and no pin it refuses. func (d *Daemon) resolveBindingRepo(b binding.Binding) (owner, repo, reason string) { if b.Owner != "" && b.Repo != "" { return b.Owner, b.Repo, "" } repos := d.Cfg.Get().Repos if len(repos) == 1 { return repos[0].Owner, repos[0].Name, "" } return "", "", fmt.Sprintf("cannot pick a repository: %d are configured; pin the binding to one", len(repos)) } // drainReactions turns available review reactions into queued remediate // runs. It runs before the task drain so a reaction queued in this pass can // be claimed by it; the consumer is built lazily once, so its drop counter // spans the process lifetime. func (d *Daemon) drainReactions(ctx context.Context) { if d.Reactions == nil || d.Store == nil { return } if d.reactionConsumer == nil { d.reactionConsumer = newReactionConsumer( // The store's owned-PR lookup is the authorization boundary: a // reaction only ever causes work against a PR archie opened and // still owns (pr-review-remediation.md decision 3). d.Store, d.Store, d.Store, d.botUserForTask, d.Log, ) } if _, err := d.reactionConsumer.drain(ctx, d.Reactions, maxReactionsPerCycle); err != nil { d.Log.ErrorContext(ctx, "reaction drain failed", "err", err) } } // botUserForTask resolves the bot login that owns task, so the reaction // consumer can exclude Archie's own comments: the global forge bot user for // root tasks. func (d *Daemon) botUserForTask(task *workflowtask.Task) string { if !d.rootTask(task) { return "" } return d.Cfg.Get().BotUser } // drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for // requeued tasks (waiting_approval approval, retry-parked) that didn't come // through a NATS publish. It returns once the stream and the claim fallback // are both empty: in-flight tasks keep running, so a task waiting on a callee // does not hold up the pass that would claim it. Shutdown waits for them // through WaitForTasks. func (d *Daemon) drainNATS(ctx context.Context) { dispatcher := d.taskDispatcher() for ctx.Err() == nil { msg, err := d.Tasks.Fetch(ctx) if err != nil && !errors.Is(err, eventbus.ErrNoMessage) { d.Log.ErrorContext(ctx, "nats fetch failed", "err", err) break } if err == nil { d.submitNATSTask(ctx, dispatcher, msg) continue } // ErrNoMessage means the stream is drained. That is precisely when // the SQLite fallback must run: requeued tasks (waiting_approval // approval, retry-parked) never went through a NATS publish, so // nothing will ever arrive on the stream to trigger them. task, claimErr := d.Store.ClaimNext(ctx) if claimErr != nil { d.Log.ErrorContext(ctx, "sqlite claim failed", "err", claimErr) break } if task == nil { break } dispatcher.Submit(ctx, task, d.process) } } // taskDispatcher returns the daemon's long-lived global dispatcher, building // it from the running config on first use. It outlives a single drain pass so // ResizeTaskDispatcher can resize the running dispatcher. func (d *Daemon) taskDispatcher() *taskDispatcher { d.dispatcherOnce.Do(func() { d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask) }) return d.dispatcher } // WaitForTasks blocks until every submitted task's process has returned. A // drain pass no longer waits for its own tasks, so shutdown is where in-flight // work is waited for. func (d *Daemon) WaitForTasks() { d.taskDispatcher().Wait() } // ResizeTaskDispatcher applies a new containers.max_concurrency to the // running global task dispatcher without a restart. Running tasks keep their // slots; a lowered limit is honoured as they finish, and a raised limit // admits queued work immediately. func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) { d.taskDispatcher().SetMaxConcurrency(maxConcurrency) } // submitNATSTask decodes a fetched message and queues it for processing. A // message that cannot be decoded is acked rather than redelivered: it will // never decode on a retry, so leaving it unacked would block the queue. func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) { envelope, err := workintake.DecodeTask(msg.Data()) if err != nil { d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err) _ = msg.Ack() return } task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity} dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) { d.processNATSTask(ctx, msg, hold) }) } // taskDispatcher limits concurrent tasks globally and runs one task per repo // at a time, in submit order, unless the repo allows concurrency. type taskDispatcher struct { allowConcurrent func(task *workflow.Task) bool mu sync.Mutex limit int active int slotsFree *sync.Cond repoTail map[string]chan struct{} pending int // holds maps a task id to the capacity its run holds, so a run waiting on // a callee can give that capacity back and take it again by task id. holds map[int64]*taskHold } // taskHold is one submitted task's claim on execution capacity: its slot under // the global limit, and its place at the tail of its repo's chain, which the // next task for that repo is blocked on. A run that waits on a callee gives // both back so the callee can run, and takes them again when the wait ends. type taskHold struct { dispatcher *taskDispatcher repo string // chain is true when the repo runs one task at a time, so the run holds a // place in that repo's chain. chain bool // baton is the run's place in its repo's chain, closed when it gives that // place up; nil while it holds none. baton chan struct{} // slot is true while the run counts against the global limit. slot bool // waits counts the callees the run is waiting on: an open call gives capacity // back and the last closed call takes it again. waits int // finished is true once the run's process returned. finished bool // id is the task this hold belongs to, named by the run once it knows the // task's id. id int64 attempt int } // name records which task's capacity this hold is, so a wait on a callee can // give it back by task id. Zero names nothing: a task arriving on the NATS // stream has no id until the store claims its row inside the run. func (h *taskHold) name(id int64, attempt int) { if id == 0 { return } h.dispatcher.mu.Lock() h.id = id h.attempt = attempt h.dispatcher.holds[id] = h h.dispatcher.mu.Unlock() } // SetMaxConcurrency accepts a new global limit without a restart. Running // tasks keep their slots; a lowered limit is enforced as they finish, and a // raised limit admits queued work immediately. func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) { d.mu.Lock() d.limit = maxConcurrency d.mu.Unlock() d.slotsFree.Broadcast() } func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher { if allowConcurrent == nil { allowConcurrent = func(*workflow.Task) bool { return false } } d := &taskDispatcher{ limit: maxConcurrency, allowConcurrent: allowConcurrent, repoTail: make(map[string]chan struct{}), holds: make(map[int64]*taskHold), } d.slotsFree = sync.NewCond(&d.mu) return d } // joinRepoLocked appends baton to the tail of repo's chain and returns the // baton ahead of it, which the joining task is blocked on. Callers hold d.mu. func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} { previous := d.repoTail[repo] d.repoTail[repo] = baton return previous } // takeSlotLocked blocks until the global limit has room, then counts the run // in. limit <= 0 means unlimited. The limit is re-read on every wake, so // SetMaxConcurrency reaches waiting tasks without recreating the dispatcher. // Callers hold d.mu. func (d *taskDispatcher) takeSlotLocked() { for d.limit > 0 && d.active >= d.limit { d.slotsFree.Wait() } d.active++ } // giveBackLocked gives back everything the run holds: its slot under the // global limit, and its place in its repo's chain, which releases the task // queued behind it. Callers hold d.mu. func (d *taskDispatcher) giveBackLocked(hold *taskHold) { if hold.slot { d.active-- hold.slot = false } if hold.baton == nil { return } close(hold.baton) if d.repoTail[hold.repo] == hold.baton { delete(d.repoTail, hold.repo) } hold.baton = nil } // finishRun retires a returned run: it holds no capacity any more, and no // Resume may take any for it. Whatever an in-flight resume takes later is // given back by reacquire, which sees finished. func (d *taskDispatcher) finishRun(hold *taskHold) { d.mu.Lock() hold.finished = true if hold.id != 0 && d.holds[hold.id] == hold { delete(d.holds, hold.id) } d.giveBackLocked(hold) d.pending-- d.mu.Unlock() d.slotsFree.Broadcast() } // setCallWaits applies a durable snapshot without accumulating duplicate events. func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) { d.mu.Lock() if d.holds[hold.id] != hold || hold.finished { d.mu.Unlock() return } previous := hold.waits hold.waits = waits if waits > 0 { d.giveBackLocked(hold) } d.mu.Unlock() if waits > 0 { d.slotsFree.Broadcast() } else if previous > 0 { // Reacquiring may block behind another task; the caller keeps running. go d.reacquire(hold) } } // reacquire returns a resumed run's capacity: its place in its repo's chain // and a slot under the global limit. The run keeps executing meanwhile -- a // wait on a callee is not a pause. A hold records one slot and one place, so a // re-acquire queued while another is still waiting gives back whichever of the // two it finds already recorded instead of taking a second. func (d *taskDispatcher) reacquire(hold *taskHold) { d.mu.Lock() if hold.finished || hold.waits > 0 { // The run returned, or started waiting again, after the reconciliation that // queued this: it needs no capacity. d.mu.Unlock() return } var previous chan struct{} // A place another re-acquire already queued is the run's one place; joining // behind it would queue a second, which the task behind it would wait on // forever. if hold.chain && hold.baton == nil { hold.baton = make(chan struct{}) previous = d.joinRepoLocked(hold.repo, hold.baton) } d.mu.Unlock() if previous != nil { <-previous } d.mu.Lock() d.takeSlotLocked() if hold.slot { // Another re-acquire took the run's one slot while this waited: give // back the slot this one just took. The place is the other's, so it is // left alone. d.active-- d.mu.Unlock() d.slotsFree.Broadcast() return } hold.slot = true if hold.finished || hold.waits > 0 { // The run ended, or started waiting again, while this was queued for // capacity: give back what it took, so no slot or place in the repo // chain leaks. d.giveBackLocked(hold) d.mu.Unlock() d.slotsFree.Broadcast() return } d.mu.Unlock() } // Submit queues a task's process behind the global limit and, unless the repo // allows concurrency, behind the task already holding that repo. process is // handed the run's capacity so it can name it once it knows the task's id and // give it back while it waits on a callee. func (d *taskDispatcher) Submit( ctx context.Context, task *workflow.Task, process func(context.Context, *workflow.Task, *taskHold), ) { repo := task.Owner + "/" + task.Repo hold := &taskHold{dispatcher: d, repo: repo} var previous chan struct{} if !d.allowConcurrent(task) { hold.chain = true hold.baton = make(chan struct{}) d.mu.Lock() previous = d.joinRepoLocked(repo, hold.baton) d.mu.Unlock() } d.mu.Lock() d.pending++ d.mu.Unlock() go func() { defer d.finishRun(hold) if previous != nil { <-previous } d.mu.Lock() d.takeSlotLocked() hold.slot = true d.mu.Unlock() process(ctx, task, hold) }() } // inFlight is how many submitted tasks have not returned yet. func (d *taskDispatcher) inFlight() int { d.mu.Lock() defer d.mu.Unlock() return d.pending } func (d *taskDispatcher) Wait() { d.mu.Lock() defer d.mu.Unlock() for d.pending > 0 { d.slotsFree.Wait() } } // poll discovers work through every organisation connection, resolved // afresh each cycle so a rotated credential or a new connection applies to // the next poll. func (d *Daemon) poll(ctx context.Context) error { d.markPoll("") // Webhook intake replaces polling; an empty pass is not a failure. if d.Cfg.Get().Forge.Intake == config.ForgeIntakeWebhook { return nil } var failed []string for _, source := range d.issueSources(ctx) { if !d.identityActive(ctx, source.Identity) { continue } cfg := d.Cfg.Get() cfg.BotUser = source.BotUser tag := d.sourceTag(source) for _, repo := range source.Repos { issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo) if complete { d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues) } else { failed = append(failed, repo.FullName()) } for _, is := range issues { if d.Tasks != nil { d.pollNATS(ctx, source.Forge, cfg, repo, is, strings.Join(is.Labels, ","), tag) } } } } return passFailed("issue poll", failed) } // passFailed reports a maintenance pass whose items failed; each item's cause // is already logged where it happened. func passFailed(pass string, failed []string) error { if len(failed) == 0 { return nil } return fmt.Errorf("%s failed for %d item(s): %s", pass, len(failed), strings.Join(failed, ", ")) } // polledIssues is one repo's eligible issues under one dispatch rule. type polledIssues struct { rule string eligible map[int]bool } // rememberPoll records a repo's latest eligible set and returns the previous // one, reporting whether it was taken under the same rule. func (d *Daemon) rememberPoll(key string, current polledIssues) (polledIssues, bool) { d.lastPolledMu.Lock() defer d.lastPolledMu.Unlock() previous, seen := d.lastPolled[key] if d.lastPolled == nil { d.lastPolled = map[string]polledIssues{} } d.lastPolled[key] = current return previous, seen && previous.rule == current.rule } // dispatchRule identifies the rule a poll selects issues by. func dispatchRule(cfg config.Config) string { return cfg.Dispatch.Trigger + "\x00" + cfg.Label + "\x00" + cfg.BotUser } // withdrawUnpolled declines the queued or running work of an identity's // issues in repo that a complete poll no longer returns: the issue was // closed, unlabelled or unassigned. func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) { if d.WithdrawTask == nil { return } eligible := make(map[int]bool, len(polled)) for _, is := range polled { eligible[is.Number] = true } // Only an issue the same rule found eligible last time and no longer // finds was withdrawn. A changed label, trigger or bot user, or the first // poll after a restart, only records the new set: tasks picked up under // the old rule were not withdrawn by anyone. previous, sameRule := d.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible}) if !sameRule { return } active, err := d.Store.TasksPage(ctx, storecontract.TaskPage{ Statuses: []string{taskstate.Queued, taskstate.Running}, Limit: 500, }) if err != nil { d.Log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err) return } for _, t := range active { if t.Owner != repo.Owner || t.Repo != repo.Name || t.Identity != identity || !t.IsForgeBacked() || t.PRNumber > 0 || eligible[t.IssueNumber] || !previous.eligible[t.IssueNumber] { continue } if err := d.WithdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil { d.Log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err) } } } // pollNATS publishes discovered issues to NATS (new flow). func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) { // Read-only existence check prevents rediscovering an existing task. existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number) if err != nil { d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err) return } if existing != nil { return } // Classify and encode here: the envelope, its routing kind and its // subject belong to the work-intake domain, not to the transport. parsed := workintake.SplitLabels(labels) task := workintake.TaskEnvelope{ Owner: repo.Owner, Repo: repo.Name, Number: is.Number, Title: is.Title, Body: is.Body, Labels: parsed, Identity: identity, Kind: workintake.KindForLabels(parsed), } if err := d.PublishTask(ctx, task); err != nil { d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err) return } d.acknowledge(ctx, fg, cfg, repo, is) } // PublishTask validates a task envelope, stamps the producing identity's // org and publishes it. The idempotency key dedups rediscovered issues. func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error { if d.Tasks == nil { return fmt.Errorf("publish task %s: no task bus configured", task.Ref()) } if err := task.Kind.Validate(); err != nil { return fmt.Errorf("publish task %s: %w", task.Ref(), err) } resolved, err := d.identityOrg(ctx, d.publisherIdentity(task.Identity)) if err != nil { return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err) } task.Org = resolved payload, err := task.Encode() if err != nil { return err } return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload) } // PublishReaction stamps the org on a review reaction, defaulting to the // root identity's, and publishes it. func (d *Daemon) PublishReaction(ctx context.Context, reaction workintake.ReviewCommentEnvelope) error { if d.Tasks == nil { return fmt.Errorf("publish reaction %s: no task bus configured", reaction.Ref()) } if reaction.Org == "" { resolved, err := d.identityOrg(ctx, d.RootIdentityID) if err != nil { return fmt.Errorf("publish reaction %s: resolve org: %w", reaction.Ref(), err) } reaction.Org = resolved } payload, err := reaction.Encode() if err != nil { return err } return d.Tasks.PublishUnique(ctx, reaction.Subject(), reaction.IdempotencyKey(), payload) } // publisherIdentity is the identity a producer publishes as: the envelope's // identity when it names one, otherwise the root identity. func (d *Daemon) publisherIdentity(ref string) identity.IdentityID { if ref == "" || identity.IdentityID(ref) == d.RootIdentityID || identity.StableID(ref) == d.RootIdentityID { return d.RootIdentityID } return identity.IdentityID(ref) } // identityOrg resolves an identity's org. No PrincipalSource means the // default org; a store error is returned. func (d *Daemon) identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) { if d.Principals == nil { return org.DefaultOrgID, nil } principal, err := d.Principals.PrincipalFor(ctx, id) if err != nil { return "", err } return principal.Org, nil } // acknowledge posts the pickup reaction and queued event // using fg -- the forge client that owns repo (identity-scoped or root) -- // and the ack reaction from cfg, the dispatch config that owns this poll. func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) { d.Log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title) if ack := cfg.Dispatch.AckReaction; ack != "" { if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil { d.Log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err) } } d.emit(ctx, events.Event{ Kind: events.KindTaskQueued, Repo: repo.FullName(), Issue: is.Number, Detail: is.Title, }) } func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool { if ok, reason, class := d.identityMayAct(ctx, name); !ok { d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject()) if class == taskstate.ParkTransient { if err := msg.Nak(); err != nil { d.Log.WarnContext(ctx, "nak failed", "err", err) } return false } if err := msg.Ack(); err != nil { d.Log.WarnContext(ctx, "ack failed", "err", err) } return false } return true } // processNATSTask decodes a NATS message, writes it to SQLite, claims it, // and runs the workflow. The message is ack'd on terminal (park is a valid // outcome). Nak on transient errors so NATS redelivers. func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) { ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header()) defer span.End() tm, err := workintake.DecodeTask(msg.Data()) if err != nil { d.Log.ErrorContext(ctx, "nats decode failed", "err", err) if err := msg.Ack(); err != nil { d.Log.WarnContext(ctx, "ack failed", "err", err) } // bad message, don't retry return } if !d.acceptTaskPublisher(ctx, msg, tm.Identity) { return } inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity) if err != nil { d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err) if err := msg.Nak(); err != nil { d.Log.WarnContext(ctx, "nats nak failed", "err", err) } return } if !inserted { // Already tracked -- dedup. Ack and move on. if err := msg.Ack(); err != nil { d.Log.WarnContext(ctx, "ack failed", "err", err) } return } task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number) if err != nil { d.Log.ErrorContext(ctx, "nats claim failed", "err", err) if err := msg.Nak(); err != nil { d.Log.WarnContext(ctx, "nats nak failed", "err", err) } return } if task == nil { // Claimed by another consumer, or task is not queued. Ack. if err := msg.Ack(); err != nil { d.Log.WarnContext(ctx, "ack failed", "err", err) } return } d.process(ctx, task, hold) if err := msg.Ack(); err != nil { d.Log.WarnContext(ctx, "ack failed", "err", err) } } // closeResolvedIssue closes a finished task's forge issue, skipping chat // tasks. Failures are logged. func (d *Daemon) closeResolvedIssue(ctx context.Context, fg forge.Forge, task *workflow.Task, comment string) { if !task.IsForgeBacked() { return } if d.ResolveRepository != nil { resolved, err := d.Repository(ctx, task, config.OperationPR) if err != nil { d.Log.WarnContext(ctx, "issue closure not permitted", "task", task.ID) return } fg = resolved.Forge } if err := fg.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil { d.Log.WarnContext(ctx, "closing resolved issue failed; it stays open and will be re-polled", "repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err) } } // reconcilePRs moves pr_open tasks to merged/rejected from GitHub state // and cleans up their worktrees. func (d *Daemon) reconcilePRs(ctx context.Context) error { tasks, err := d.Store.OpenPRs(ctx) if err != nil { return fmt.Errorf("reconcile query: %w", err) } var failed []string for _, t := range tasks { fg := d.forgeFor(ctx, &t) if _, unavailable := fg.(*forge.NoopForge); unavailable { continue } trees := d.treesFor(&t) // PRState/PR reconciliation is valid regardless of Source -- a // chat-sourced task can still open a real PR against a real // branch. Only the issue-label call (tied to the synthetic // issue number) is forge-only. state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber) if err != nil { d.Log.WarnContext(ctx, "PR state check failed", "pr", t.PRNumber, "err", err) failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) continue } attempt := t.Attempt switch state { case "merged": if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, ""); err != nil { d.Log.ErrorContext(ctx, "PR merged but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err) failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) continue } if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil { d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err) } // Close the issue explicitly; nothing else does. d.closeResolvedIssue(ctx, fg, &t, fmt.Sprintf( "Closed by #%d, merged by archie.", t.PRNumber, )) d.Log.InfoContext(ctx, "PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) d.emit(ctx, events.Event{ Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt, Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, Data: map[string]any{"pr": t.PRNumber}, }) case "closed": if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge"); err != nil { d.Log.ErrorContext(ctx, "PR rejected but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err) failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber)) continue } if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil { d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err) } d.Log.InfoContext(ctx, "PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber) d.emit(ctx, events.Event{ Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt, Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber, Data: map[string]any{"pr": t.PRNumber}, }) } } return passFailed("PR reconcile", failed) } func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) { // A task off the NATS stream only has an id once the store has claimed its // row, inside this call, so its capacity is named here. hold.name(task.ID, task.Attempt) // The workflow may name the identity the run acts as, which everything // below is chosen by, so it is settled before anything else. if !d.adoptWorkflowIdentity(ctx, task) { return } // Register first so the task can be stopped during setup. ctx, finished := d.running.begin(ctx, task.ID, task.Identity) defer finished() defer d.openTaskLog(task)() if d.parkIfIdentityUnresolvable(ctx, task) { return } trees, repo, err := d.taskWorkspace(ctx, task) if err != nil { d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err)) return } if d.ContainerPool == nil || d.Tasks == nil { d.parkCapabilityUnavailable(ctx, task) return } d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt) workDir, ok := d.prepareWorkspace(ctx, task, trees, repo) if !ok { return } if !task.HasRepository() { defer func() { if err := trees.RemoveScratch(task.ID); err != nil { d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err) } }() } // The workflow is pinned before the container starts, because the // profile it names decides the container's image. profile, ok := d.pinTaskProfile(ctx, task) if !ok { return } if profile.IsKit() { d.runKitTask(ctx, task, repo, workDir, profile) d.teardownStorage(ctx, task, repo, workDir) d.cleanupTerminalTaskWorktree(ctx, task, trees) return } ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image) if !ok { return } defer d.ContainerPool.Release(ctx, ctr) defer revokeCredential() // Run the whole task in archie-agent, which calls back to the daemon for // store, forge and push operations. limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std()) runCtx, stopWatch := withContainerExit(limitCtx, ctr) d.runViaAgent(runCtx, task, repo, profile, nil, credential) stopWatch() stopLimit() d.teardownStorage(ctx, task, repo, workDir) d.cleanupTerminalTaskWorktree(ctx, task, trees) } // teardownStorage runs after the workflow completes. The Docker backend is // a no-op; future backends (temp volumes, NFS leases) use this hook. func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) { if d.Storage == nil { return } if err := d.Storage.Teardown(ctx, storage.TaskRef{ WorktreeDir: workDir, Ecosystem: repo.Ecosystem, PersistentStorage: repo.PersistentStorage, Owner: task.Owner, Repo: task.Repo, }); err != nil { d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err) } } func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) { if task.HasRepository() && d.ResolveRepository != nil { resolved, err := d.Repository(ctx, task, config.OperationClone) if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) { // Checked before the run rather than at its commit step, so a // missing permit parks in seconds instead of after the agent's work. _, err = d.Repository(ctx, task, config.OperationPush) } return resolved.Trees, resolved.Repository, err } trees := d.treesFor(task) repo, ok := d.repoFor(task) if !ok && task.HasRepository() { return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")} } return trees, repo, nil } // prepareWorkspace makes the directory the container binds as its // workspace, which must exist before Acquire: a fresh clone of the task's // repository, or an empty scratch directory for a task with none. It parks // the task and reports false when it cannot. func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) { if !task.HasRepository() { dir, err := trees.PrepareScratch(task.ID) if err != nil { d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err) d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient) return "", false } return dir, true } // The retry mode on the task decides whether to resume the pushed branch or // reset onto base. Prepare clones a missing worktree either way. target := worktree.Fresh if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork { if task.Branch == "" { // The action refuses this combination, so an empty branch here is a // dispatch bug upstream and no retry can fix it. d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID) d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision) return "", false } target = worktree.Target(task.Branch) } // Every task gets an independent full clone. prepare := func() (string, string, error) { return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target) } dir, branch, err := prepare() if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) { dir, branch, err = prepare() } if err != nil { reason := "worktree prepare failed: " + err.Error() if target != worktree.Fresh { reason = "worktree resume failed: " + err.Error() } d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err) d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) return "", false } task.Branch = branch if err := d.Store.Update(ctx, task); err != nil { d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err) } return dir, true } // cleanupTerminalTaskWorktree removes a settled task's worktree, unless it // holds uncommitted work or its state cannot be read. func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) { if d.Store == nil || trees == nil || task == nil || !task.HasRepository() { return } cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() latest, err := d.Store.TaskByID(cleanupCtx, task.ID) if err != nil { d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err) return } if latest == nil { return } // Settled statuses. Keep the worktree while a PR exists. switch latest.Status { case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted: if latest.PRNumber != 0 { return } if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) { return } if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil { d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err) } } } // worktreeHoldsUncapturedWork reports whether a terminal task's worktree has // to survive cleanup because it may hold work no commit captured. A worktree // that is not on disk at all has nothing to keep, so a task whose clone was // already removed still cleans up whatever leftovers share its path. func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool { dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber) if _, err := os.Stat(dir); err != nil { if errors.Is(err, fs.ErrNotExist) { return false } d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err) return true } uncommitted, err := trees.HasUncommittedChanges(ctx, dir) if err != nil { d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err) return true } if uncommitted { d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir) return true } return false } // openTaskLog opens the task's log sink for one process() call and returns // its closer. Failure to open is logged, not fatal. func (d *Daemon) openTaskLog(task *workflow.Task) func() { if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil { d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err) } return func() { if err := d.TaskLogs.Close(task.ID); err != nil { d.Log.Warn("task log sink close failed", "task", task.ID, "err", err) } } } // acquireTaskContainer prepares and acquires the sandbox container for a // task, reporting false when it could not -- having already parked the // task and logged why. The caller releases the container it returns. func (d *Daemon) acquireTaskContainer( ctx context.Context, task *workflow.Task, repo config.Repo, workDir string, image string, ) (*container.Container, string, func(), bool) { park := func(reason string, err error) { d.Log.ErrorContext(ctx, reason, "err", err) d.parkRunningTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient) } // Resolve the PR number from inputs before the brief is written. if task.Workflow == "pr-review" { task.PRNumber = task.EffectivePRNumber() } if err := writeTaskBrief(workDir, task); err != nil { park("task.json write failed", err) return nil, "", nil, false } // Fetch the pull request with the daemon's forge credential into workDir, // so the container reads it without credentials. if task.Workflow == "pr-review" { if err := prefetchPRReview(ctx, d.forgeFor(ctx, task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil { park("pr-review prefetch failed", err) return nil, "", nil, false } } // Guard: Storage may be nil if the daemon was wired incorrectly. In // normal operation, Storage is always set when ContainerPool is set. if d.Storage == nil { d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container") d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient) return nil, "", nil, false } mounts, err := d.Storage.Setup(ctx, storage.TaskRef{ WorktreeDir: workDir, Ecosystem: repo.Ecosystem, PersistentStorage: repo.PersistentStorage, Owner: task.Owner, Repo: task.Repo, }) if err != nil { park("storage setup failed", err) return nil, "", nil, false } credential, revokeCredential, err := d.runCredential(ctx, task) if err != nil { d.teardownStorage(ctx, task, repo, workDir) park("run credential failed", err) return nil, "", nil, false } model, closeModel, err := d.openModelEgress(ctx, task, credential) if err != nil { revokeCredential() d.teardownStorage(ctx, task, repo, workDir) park("model egress failed", err) return nil, "", nil, false } revokeRun := revokeCredential revokeCredential = func() { closeModel(); revokeRun() } // The container lives in its own network namespace, so a loopback State // Store target names the container itself. Give this task an address on // the host bridge that forwards to it, closed with the task. stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential) if err != nil { revokeCredential() d.teardownStorage(ctx, task, repo, workDir) park("state store relay failed", err) return nil, "", nil, false } revokeCredential = release ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...)) if err != nil { revokeCredential() // Roll back the storage we just set up: the mounts were created // for this task but no container will use them. Leaking them until // a later TTL sweep is not acceptable on a backend that allocates // real resources (volumes, NFS leases). d.teardownStorage(ctx, task, repo, workDir) park("container acquire failed", err) return nil, "", nil, false } d.admitToRelay(ctx, relay, task, ctr) return ctr, credential, revokeCredential, true } // writeTaskBrief writes task.json, the container's boot-time brief. func writeTaskBrief(workDir string, task *workflow.Task) error { return container.WriteTaskJSON(workDir, container.TaskPayload{ ID: task.ID, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber, Title: task.Title, Body: task.Body, Labels: strings.Split(task.Labels, ","), Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan, Inputs: task.Inputs, }) } // parkCapabilityUnavailable parks a task whose daemon-side capability is // missing (container pool or task transport): both are environmental, so // the park class is transient and the KindParked event carries the class // for the dashboard. The reason distinguishes the two in logs and events. func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) { var reason string if d.ContainerPool == nil { reason = "managed agent container pool is unavailable; refusing to run this task on the host" d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID, "hint", "enable containers and make the archie-agent image available, then retry the task") } else { reason = "agent task transport is unavailable; refusing to run this task on the host" d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID) } d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) } // parkRunningTask moves a running task to parked with class, using a // bounded context that ignores cancellation. ErrStaleTransition is ignored; // other errors are logged. func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) { d.parkBlockedTask(ctx, task, reason, class, "") } // parkBlockedTask is parkRunningTask recording the blocker the park site // knows, so the dashboard can link to its fix. func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) { if d.Store == nil { return } writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil { if !errors.Is(err, storecontract.ErrStaleTransition) { d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err) } return } // Recorded only once the transition landed. An entry claiming the task // parked would be a lie when another actor moved it out of running first // -- the same race the parked event is withheld for. d.recordPark(ctx, task.ID, reason) d.emit(ctx, events.Event{ Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt, Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow, Detail: reason, Data: map[string]any{"park_class": class}, }) } // recordPark writes a park reason to the attempt's log. Dropped when no sink // is open. func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) { d.TaskLogs.Write(ctx, taskID, logging.Entry{ Time: time.Now(), Level: slog.LevelError.String(), Message: "task parked: " + reason, Fields: map[string]any{"component": "daemon", "task": taskID}, }) } // runViaAgent hands a task to archie-agent and waits for its result. It parks // the task only when the agent never answered or failed before recording an // outcome. // // credential is the task's run credential; the agent presents it to push the // task's branch. func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, harness *agentrun.HarnessSpec, credential string) { cfg, err := d.taskConfig(ctx, task) if err != nil { d.parkRunningTask(ctx, task, err.Error(), taskstate.ParkTransient) return } taskCfg := cfg.ForTask() d.captureAttemptConfig(ctx, task, taskCfg) var instructions workflow.Instructions if d.TaskInstructions != nil { instructions.Instance, instructions.Organisation, err = d.TaskInstructions.TaskInstructions(ctx, task.ID) if err != nil { d.parkRunningTask(ctx, task, "resolve the task's instructions: "+err.Error(), taskstate.ParkTransient) return } } req := taskrun.Request{ Instructions: instructions, Task: task, Repo: repo, Cfg: taskCfg, Providers: agentexec.ProvidersFromConfig(cfg.Providers), MCPServers: cfg.Tools.MCPServers, RunCredential: credential, WorkflowDefinition: task.WorkflowDefinitionYAML, Tools: d.toolAllowlist(ctx, task, profile.Tools), Harness: harness, } req.Trace = map[string]string{} otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace)) data, err := json.Marshal(req) if err != nil { d.Log.ErrorContext(ctx, "taskrun encode failed", "task", task.ID, "err", err) d.parkRunningTask(ctx, task, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient) return } reply, err := d.requestTaskRun(ctx, task.ID, data) if err != nil { if stoppingOnly(ctx) { d.Log.InfoContext(ctx, "daemon stopping; task left running for the next daemon to re-queue", "task", task.ID) return } err = runFailure(ctx, err) d.Log.ErrorContext(ctx, "taskrun request failed", "task", task.ID, "err", err) d.parkRunningTask(ctx, task, "taskrun request failed: "+err.Error(), taskstate.ParkTransient) return } var resp taskrun.Response if err := json.Unmarshal(reply, &resp); err != nil { d.Log.ErrorContext(ctx, "taskrun decode response failed", "task", task.ID, "err", err) d.parkRunningTask(ctx, task, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient) return } if resp.Error != "" { d.Log.ErrorContext(ctx, "taskrun run failed", "task", task.ID, "err", resp.Error) d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient) return } // The agent applies the task's own outcome transition, so a status the // task cannot reach from running means nothing landed: empty, unknown, or // still running. Transient, because the cause is environmental -- a State // Store that refused the transition -- and a retry can record one. if !taskstate.CanTransition(taskstate.Running, resp.Status) { reason := noOutcomeReason(resp.Status) d.Log.ErrorContext(ctx, "taskrun run recorded no outcome", "task", task.ID, "status", resp.Status) d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient) return } // Reached only past every failure return above, so this is always a // fully decoded, error-free response -- a version is never recorded off // a request that never reached an agent, or a response describing a // parked/errored task. if d.AgentStatus != nil && resp.AgentVersion != "" { d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType) } d.Log.InfoContext(ctx, "taskrun complete", "task", task.ID, "status", resp.Status) } // noOutcomeReason names why a taskrun reply left the task without an outcome, // quoting the status so the log line points at what the agent actually said. func noOutcomeReason(status string) string { if status == "" { return "agent returned no status: the run could not record an outcome" } return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status) } // pinTaskProfile pins the task's workflow definition and resolves its agent // profile, parking the task on failure. func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) { if err := d.pinWorkflowDefinition(ctx, task); err != nil { d.Log.ErrorContext(ctx, "pin workflow definition failed", "task", task.ID, "err", err) d.parkBlockedTask(ctx, task, "pin workflow definition: "+err.Error(), pinParkClass(err), parkBlocker(err)) return config.AgentProfile{}, false } iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML) if err == nil { // Every producer meets the declared interface here, including the // ones that cannot read definitions when they enqueue (chat and // dashboard spawns go through the Gateway, which never does). err = iface.CheckInputs(task.Inputs) } if err == nil { var profile config.AgentProfile if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil { if err = validateProfileMeetsNeeds(profile, iface.Needs()); err == nil { return profile, true } } } d.Log.ErrorContext(ctx, "agent profile unavailable", "task", task.ID, "err", err) d.parkRunningTask(ctx, task, "workflow "+task.Workflow+": "+err.Error(), taskstate.ParkNeedsDecision) return config.AgentProfile{}, false } // validateProfileMeetsNeeds rejects a Kit profile whose harness cannot serve // the captures the workflow needs. Image profiles always pass. func validateProfileMeetsNeeds(profile config.AgentProfile, needs workflowtask.WorkflowNeeds) error { if !profile.IsKit() || (!needs.Captures && len(profile.Tools) == 0) { return nil } adapter, ok := agentexec.LookupHarnessAdapter(profile.Adapter) if !ok || len(adapter.MCPConfig) == 0 { return fmt.Errorf("profile adapter %q cannot register the required Archie MCP tools", profile.Adapter) } return nil } // pinWorkflowDefinition ensures the task carries a valid definition pin for // the workflow it names and that its org has the workflow enabled. Every run // passes through here, whatever started it, so this is the one enablement // check for issue labels, handoffs, approvals, calls and playbooks alike. func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error { ctx, err := d.workflowContext(ctx, task) if err != nil { return err } if err := d.pinDefinition(ctx, task); err != nil { return err } enablement, err := d.workflowEnablement(ctx) if err != nil { return pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow enablement: %w", err)} } if !enablement.Enabled(task.Org, task.Workflow) { return blockedPark{taskstate.BlockerWorkflowDisabled, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow %s is disabled", task.Workflow)}} } if err := d.checkPackageTrigger(ctx, task); err != nil { return pinFailure{taskstate.ParkNeedsDecision, err} } return nil } // pinDefinition keeps a valid pin and resolves a missing or mismatched one // again from the active definitions. func (d *Daemon) pinDefinition(ctx context.Context, task *workflow.Task) error { pinned, ok, err := workflow.PinnedDefinitionID(task) if err != nil { return pinFailure{taskstate.ParkNeedsDecision, err} } if ok && pinned == task.Workflow { return nil } d.runActionPlaybooks(ctx, task) if d.WorkflowDefinitions == nil { return d.pinWorkflowFromCollection(ctx, task, workflow.ShippedDefinitions(), 0) } collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx) if err != nil { return pinFailure{taskstate.ParkTransient, err} } return d.pinWorkflowFromCollection(ctx, task, collection, version) } // pinFailure carries the park class for a pin failure: operator-actionable // for a bad pin or missing workflow, transient for a store error. type pinFailure struct { class taskstate.ParkClass err error } func (e pinFailure) Error() string { return e.err.Error() } func (e pinFailure) Unwrap() error { return e.err } // pinParkClass reports the class a pin failure carries, defaulting to // taskstate.ParkNeedsDecision for a cause that did not classify itself: the safe // misread is "an operator should look at this", never "a requeue will fix it". func pinParkClass(err error) taskstate.ParkClass { if failure, ok := errors.AsType[pinFailure](err); ok { return failure.class } return taskstate.ParkNeedsDecision } // blockedPark names what a park is missing alongside the failure. type blockedPark struct { blocker taskstate.Blocker err error } func (e blockedPark) Error() string { return e.err.Error() } func (e blockedPark) Unwrap() error { return e.err } // parkBlocker reports the blocker err names, or empty. func parkBlocker(err error) taskstate.Blocker { if b, ok := errors.AsType[blockedPark](err); ok { return b.blocker } if failure, ok := errors.AsType[*forge.AccessError](err); ok && failure.Code == "operation_denied" { return taskstate.BlockerConnectionPermit } return "" } // runActionPlaybooks runs the action playbooks matching a task before it is // pinned. Failures never fail the task; tasks already naming a workflow are // skipped. func (d *Daemon) runActionPlaybooks(ctx context.Context, task *workflow.Task) { if d.Playbooks == nil || d.PlaybookLedger == nil || task.Workflow != "" { return } if err := d.Playbooks.Run(ctx, d.PlaybookLedger, d.Log, playbookInput(task)); err != nil { d.Log.WarnContext(ctx, "action playbook run failed", "task", task.ID, "err", err) } } // resolveWorkflowID picks the definition to pin: a matching playbook, then // the kind defaults. A task already naming a workflow keeps it. func (d *Daemon) resolveWorkflowID(ctx context.Context, task *workflow.Task, available map[string]struct{}) (string, error) { if d.Playbooks == nil || task.Workflow != "" { return workflow.ResolveWorkflowID(task, available) } decision, matched := d.Playbooks.Dispatch(ctx, playbookInput(task)) if !matched { return workflow.ResolveWorkflowID(task, available) } if _, defined := available[decision.Workflow]; !defined { // Reported, not silently downgraded to the binding's choice: the // operator bound this trigger to a named workflow, and running a // different one under their rule is worse than parking the task. return "", fmt.Errorf("playbook %q binds this trigger to workflow %q, which is not defined", decision.PlaybookID, decision.Workflow) } d.Log.InfoContext(ctx, "playbook selected workflow", "task", task.ID, "playbook", decision.PlaybookID, "playbook_version", decision.Version, "workflow", decision.Workflow) return decision.Workflow, nil } // playbookInput builds the coordinator's dispatch input from a task row. The // event surface is what the intake point actually knows about the originating // issue -- the fields a `when` condition can read; it is deliberately not // padded with values the daemon would have to invent. func playbookInput(task *workflow.Task) playbook.DispatchInput { labels := workintake.SplitLabels(task.Labels) kind := string(workintake.KindForLabels(labels)) return playbook.DispatchInput{ Labels: labels, Kind: kind, TaskID: workintake.TaskEnvelope{Org: task.Org, Identity: task.Identity, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber}.IdempotencyKey(), Event: map[string]any{ "kind": kind, "labels": labels, "owner": task.Owner, "repo": task.Repo, "number": task.IssueNumber, "title": task.Title, "body": task.Body, "source": task.Source, }, } } func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.Task, collection workflow.WorkflowDefinitionCollection, version int64) error { available := make(map[string]struct{}, len(collection.Definitions)) for _, definition := range collection.Definitions { available[definition.ID] = struct{}{} } id, err := d.resolveWorkflowID(ctx, task, available) if err != nil { return pinFailure{taskstate.ParkNeedsDecision, err} } definition, ok := collection.DefinitionByID(id) if !ok { return pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow definition %q disappeared", id)} } task.Workflow = id task.WorkflowDefinitionVersion = version task.WorkflowDefinitionYAML = definition.YAML task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML) if err := d.Store.Update(ctx, task); err != nil { return pinFailure{taskstate.ParkTransient, fmt.Errorf("persist workflow definition pin: %w", err)} } return nil } const ( // defaultTaskRunReadyTimeout is how long requestTaskRun retries a // nats.ErrNoResponders before giving up, when Daemon.TaskRunReadyTimeout // is unset. defaultTaskRunReadyTimeout = 20 * time.Second // defaultTaskRunRetryBackoff is the delay between retry attempts, when // Daemon.TaskRunRetryBackoff is unset. defaultTaskRunRetryBackoff = 250 * time.Millisecond ) func (d *Daemon) taskRunReadyTimeout() time.Duration { if d.TaskRunReadyTimeout > 0 { return d.TaskRunReadyTimeout } return defaultTaskRunReadyTimeout } func (d *Daemon) taskRunRetryBackoff() time.Duration { if d.TaskRunRetryBackoff > 0 { return d.TaskRunRetryBackoff } return defaultTaskRunRetryBackoff } // requestTaskRun publishes the taskrun request, retrying on // nats.ErrNoResponders until the container subscribes. Other errors return // at once. func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) { subject := agentnats.SubjectForTask(taskID) deadline := time.Now().Add(d.taskRunReadyTimeout()) backoff := d.taskRunRetryBackoff() for { reply, err := d.Tasks.Request(ctx, subject, data) if err == nil { return reply, nil } if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) { return nil, err } wait := min(backoff, time.Until(deadline)) if wait <= 0 { return nil, err } select { case <-ctx.Done(): return nil, ctx.Err() case <-time.After(wait): } } } // taskStateStoreEndpoint is the State Store address one task container is // handed, the relay carrying it (nil when none is needed) and the release that // ends that relay. The container lives // in its own network namespace, so a loopback target names the container // itself; behind a relay, the address is the host bridge's. func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) { relay, err := d.openTaskStateStoreRelay(ctx) if err != nil { return "", nil, release, err } if relay == nil { return d.ConnectedStateStore.URL, nil, release, nil } return relay.Addr(), relay, func() { relay.Close(); release() }, nil } // admitToRelay lets ctr, and only ctr, through relay. A container whose // addresses cannot be read is refused rather than let anyone through. func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) { if relay == nil { return } addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID) if err != nil { d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err) } relay.Admit(addrs) } // openTaskStateStoreRelay starts the relay that carries one task container's // State Store traffic to the daemon's target. The target stays bound where the // operator put it; the container is handed an address on the host bridge, the // same way it is handed NATS's. Nil when the target already reaches the // containers, or when no bridge address is known. func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) { addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL) if !loopback { return nil, nil } gateway := "" if d.ContainerPool != nil { gateway = d.ContainerPool.HostGateway() } if gateway == "" { d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it", "target", d.ConnectedStateStore.URL) return nil, nil } return openStateStoreRelay(ctx, gateway, addr) } // containerEnv is the environment a task container starts with. stateStoreURL // is the address the container can reach the State Store by, which is the // relay's rather than the daemon's own target. func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string { var env []string // The endpoint the daemon's own client connected with at startup, not // the live config: a reloaded [nats] section must not point new // containers at a server the daemon is not publishing on. env = append(env, "NATS_URL="+d.ConnectedNATS.URL) if token := d.ConnectedNATS.ContainerToken(); token != "" { env = append(env, "NATS_TOKEN="+token) } // Pass the container-reachable endpoint with this task's scoped token, // never the daemon's own. if stateStoreURL != "" { env = append(env, "STATE_STORE_URL="+stateStoreURL) if len(d.ConnectedStateStore.CA) > 0 { env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA)) } if stateStoreToken != "" { env = append(env, "STATE_STORE_TOKEN="+stateStoreToken) } } // The agent runs as root in the container, so its commits land owned by // UID 0 on the host. The daemon's UID/GID let the agent hand the worktree // back before the daemon reads it to push. env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid())) env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid())) return env } // runCredential issues the task's run credential, or "" when no State Store // is configured. It fails when no issuer is wired. func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) { if d.ConnectedStateStore.URL == "" { return "", func() {}, nil } if d.RunCredentials == nil { return "", nil, fmt.Errorf("state store is configured but no run credential issuer is wired") } return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std()) } func (d *Daemon) configFor(task *workflow.Task) config.Config { cfg := d.Cfg.Get() if !d.rootTask(task) { cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{} cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil } return cfg } // captureAttemptConfig records the attempt's effective configuration as an // event. Failures are logged. func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) { if d.Store == nil { return } document, err := json.Marshal(cfg) if err != nil { d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err) return } event := events.Event{ At: time.Now().UTC(), Kind: events.KindConfigCaptured, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo, Issue: task.IssueNumber, Workflow: task.Workflow, Attempt: task.Attempt, Data: map[string]any{ "schema": events.ConfigCapturedSchema, // Raw so the document is embedded once, as it was encoded, rather // than as a string a reader would have to decode again. "document": json.RawMessage(document), }, } id, err := d.Store.InsertEvent(ctx, event) if err != nil { d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err) return } // The assigned ID tells the event sink to broadcast without inserting a // duplicate row. event.ID = id d.emit(ctx, event) } // identityMayAct reports whether a task's identity may act, and the park // class if not. Empty is the root identity. An unknown or inactive identity // never falls back to the root forge. func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) { id := d.publisherIdentity(name) if d.IdentityRepository != nil { value, err := d.IdentityRepository.Get(ctx, id) if err != nil { return false, "execution identity unavailable: " + err.Error(), taskstate.ParkTransient } if value.Lifecycle != identity.LifecycleActive { return false, "identity may not act", taskstate.ParkNeedsDecision } return true, "", "" } if name == "" || name == string(d.RootIdentityID) { return true, "", "" } return false, "identity " + name + " no longer resolves", taskstate.ParkTransient } // parkIfIdentityUnresolvable parks task when its non-empty identity cannot be // acted on, reporting whether it did. An empty identity is the root // single-identity path and never parks here. func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool { ok, reason, class := d.identityMayAct(ctx, task.Identity) if ok { return false } d.parkRunningTask(ctx, task, reason, class) return true } func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool { if d.IdentityRepository == nil { return true } if id == "" { d.Log.ErrorContext(ctx, "identity lifecycle check failed", "err", "identity ID is empty") return false } value, err := d.IdentityRepository.Get(ctx, id) if err != nil { d.Log.ErrorContext(ctx, "identity lifecycle check failed", "identity", id, "err", err) return false } return value.Lifecycle == identity.LifecycleActive } // forgeFor returns the forge client that owns task: its organisation // connection, else the root d.Forge for the root alone. This is the safety // boundary that keeps one identity's forge token from being used against // another identity's repos. func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge { if d.ResolveRepository != nil && task.HasRepository() { resolved, err := d.Repository(ctx, task, config.OperationClone) if err != nil { return forge.NewNoop(d.Log) } return resolved.Forge } if !d.rootTask(task) { return forge.NewNoop(d.Log) } return d.Forge } // treesFor returns the worktree manager that owns task, mirroring forgeFor. func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager { if d.ResolveRepository != nil && task.HasRepository() { // Cleanup needs only the org's path, not a still-valid external credential. return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())} } if !d.rootTask(task) { return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))} } return d.Trees } func (d *Daemon) rootTask(task *workflow.Task) bool { return task.Identity == "" || identity.IdentityID(task.Identity) == d.RootIdentityID } // repoFor resolves a root task's repo config from Cfg.Repos. func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) { if !d.rootTask(t) { return config.Repo{}, false } for _, r := range d.Cfg.Get().Repos { if r.Owner == t.Owner && r.Name == t.Repo { return r, true } } return config.Repo{}, false } // allowConcurrentForTask reports whether the task's repo allows concurrent // tasks. Unknown repos do not. func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool { if !task.HasRepository() { return true } repo, ok := d.repoFor(task) return ok && repo.AllowConcurrent } // LastPollAt reports when the least recently polled loop last began a pass, // or the zero time when no pass has started yet. func (d *Daemon) LastPollAt() time.Time { var oldest int64 d.lastPollAt.Range(func(_, v any) bool { if ns, ok := v.(int64); ok && (oldest == 0 || ns < oldest) { oldest = ns } return true }) if oldest == 0 { return time.Time{} } return time.Unix(0, oldest).UTC() } // markPoll stamps the start of a poll pass for one loop, so a hung pass shows // as stale. func (d *Daemon) markPoll(loop string) { d.lastPollAt.Store(loop, time.Now().UnixNano()) } // errContainerExited is the cancellation cause when a task's agent container // stops before answering its taskrun request. var errContainerExited = errors.New("agent container exited before the task finished") // runFailure names why a taskrun request failed: the container's exit or the // time limit when either cancelled it, else the request's own error. func runFailure(ctx context.Context, err error) error { if cause := context.Cause(ctx); errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit) { return cause } return err } // exitWatcher is a running agent container, seen only for how it ends. type exitWatcher interface { Exited() <-chan struct{} ExitDetail() string } // withContainerExit derives a context cancelled when exited closes. A dead // container never answers its taskrun request, so without this the request // blocks forever and the task row stays running. func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) { runCtx, cancel := context.WithCancelCause(ctx) go func() { select { case <-ctr.Exited(): if detail := ctr.ExitDetail(); detail != "" { cancel(fmt.Errorf("%w: %s", errContainerExited, detail)) return } cancel(errContainerExited) case <-runCtx.Done(): } }() return runCtx, func() { cancel(nil) } } // errTaskTimeLimit is the cancellation cause when a task outruns its overall // time limit (the execution settings' max task runtime). var errTaskTimeLimit = errors.New("task exceeded its time limit") // withTaskTimeLimit bounds a whole task run. A limit of zero or less applies // no bound. func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) { if limit <= 0 { return ctx, func() {} } return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit)) } // RemoveWorktree deletes a task's clone from the trees of the identity that // owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold // uncaptured work, and the operator's archive is the decision to discard it. func (d *Daemon) RemoveWorktree(task *workflow.Task) error { trees := d.treesFor(task) if trees == nil { return nil } ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber) } // cleanupWorktree removes a task's clone, first handing back any root-owned // files a killed container left in it. func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error { err := trees.Cleanup(owner, repo, issue) if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) { return trees.Cleanup(owner, repo, issue) } return err } // reownDenied repairs dir when err says the daemon was refused access to it, // and reports whether the caller should retry. A SIGKILLed task container // leaves root-owned files that only a container can chown back. func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool { if err == nil || !errors.Is(err, fs.ErrPermission) || d.ContainerPool == nil { return false } if rerr := d.ContainerPool.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil { d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr) return false } return true } // InFlight counts the same submitted work that graceful shutdown waits for, // including tasks waiting for a repository or concurrency slot. func (d *Daemon) InFlight() int { dispatcher := d.taskDispatcher() dispatcher.mu.Lock() defer dispatcher.mu.Unlock() return dispatcher.pending }