Files
archie/internal/daemon/daemon.go
T
sam f92c5b4536
ci/woodpecker/pr/quality Pipeline was successful
Quality / check (pull_request) Canceled after 7m39s
chore(runtime): delete Kit and external agent harness support (#1877)
Archie runs its own agent loop only. Removes Kit resolution and launch,
the harness runner and adapters, the setup terminal and sandbox view,
harness secrets and OAuth capture (table dropped in migration 0075), the
task-tools contract that only Kit containers reached, and the
sandbox-kit-spec dependency. Agent profiles carry image and tools only.
The egress proxy keeps the model-session path; the sandbox relay code
stays for #1867.
2026-11-12 21:29:05 +11:00

2835 lines
105 KiB
Go

// Package daemon is archied's resident loop: poll GitHub for labelled
// issues, enqueue them, and process tasks through their routed workflows.
// State lives in the store; the daemon is restartable at any point.
package daemon
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"log/slog"
"maps"
"os"
"path/filepath"
"strings"
"sync"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/eda/playbook"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/eventbus"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
"github.com/samcharles93/archie-core/internal/logging"
"github.com/samcharles93/archie-core/internal/natsrpc"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/taskrun"
"github.com/samcharles93/archie-core/internal/taskstate"
"github.com/samcharles93/archie-core/internal/worktree"
)
// TaskBus publishes discovered work, collects it back, and asks a worker to
// run a task.
type TaskBus interface {
PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error
Fetch(ctx context.Context) (eventbus.Message, error)
Request(ctx context.Context, subject string, payload []byte) ([]byte, error)
}
// RunCredentialIssuer issues the one credential a task's container holds: it
// authorizes the task's own State Store calls and the push of its branch.
// It lapses at the task's time limit plus a margin if never revoked.
type RunCredentialIssuer interface {
Issue(ctx context.Context, task *workflow.Task, limit time.Duration) (token string, revoke func(), err error)
}
// NATSEndpoint is the broker address and credential archied connected with at
// startup. It is runtime state rather than configuration: embedded mode
// generates both values, and external secret references are resolved once.
type NATSEndpoint struct {
URL string
Token string
// Scoped is true on the embedded broker, where a container logs in with
// its run credential and reaches only its own task's subjects.
Scoped bool
}
// ContainerToken is the broker token a container is handed: none on a
// scoped broker. An external broker cannot scope a container, so it still
// gets the broker's token (reported as a health issue by archied).
func (e NATSEndpoint) ContainerToken() string {
if e.Scoped {
return ""
}
return e.Token
}
// StateStoreEndpoint is the State Store target and the CA its certificate
// chains to, both passed to agent containers.
type StateStoreEndpoint struct {
URL string
CA []byte
}
type Daemon struct {
ResolveRepository func(context.Context, *workflow.Task, string) (RepositoryAccess, error)
// Cfg publishes the running configuration. Read through Cfg() so a
// reload swaps the published snapshot atomically. See config.Holder.
Cfg *config.Holder
// ConnectedNATS is the endpoint the daemon connected with at startup;
// container env uses it, never live config.
ConnectedNATS NATSEndpoint
// ConnectedStateStore is the State Store target the daemon dialed at
// startup, passed to containers.
ConnectedStateStore StateStoreEndpoint
Store storecontract.TaskStore
// Mappings persists payload field mappings.
// Used by the binding dispatch loop to resolve capture bodies against
// the mapping a binding names. Optional: nil disables the binding
// dispatch loop (legacy behaviour).
Mappings storecontract.MappingStore
// Bindings persists playbook bindings.
// Optional: nil disables the binding dispatch loop (legacy behaviour).
Bindings storecontract.BindingStore
// BindingDispatcher lists undispatched captures, finds armed bindings and
// records dispatches. Nil disables binding dispatch.
BindingDispatcher storecontract.BindingDispatcher
// MappingMatches counts each event a mapping resolved at dispatch.
// Optional: nil leaves match counts unrecorded.
MappingMatches storecontract.MappingMatchRecorder
// BindingTaskCreator enqueues a task triggered by a binding. Split
// off TaskLifecycle so the lifecycle surface stays narrow and the
// dispatch loop does not acquire the full task-creation contract.
// Optional: nil disables the dispatch loop.
BindingTaskCreator storecontract.BindingTaskCreator
// Access evaluates the policy chain at dispatch, the second of the two
// Authorizer call sites: the workflow's identity may `run` the workflow in
// its workspace. Optional: nil dispatches without the chain, which is the
// behaviour of an install that has not built it.
Access access.Authorizer
// Principals assembles the dispatch principal for the workflow's
// identity. Wired with Access or not at all.
Principals access.PrincipalSource
// Denials records dispatch refusals. Optional: nil skips the record.
Denials access.DenialRecorder
Forge forge.Forge
Trees *worktree.Manager
Bus *events.Bus
Log *slog.Logger
// Tasks is the NATS task bus. Always set in production.
Tasks TaskBus
// reactionConsumer is built lazily by drainReactions and kept for its
// lifetime drop counter.
reactionConsumer *reactionConsumer
// lastPolled remembers each repo's eligible issues from its previous
// complete poll, under the dispatch rule that produced them, so only an
// issue that was eligible and dropped out reads as withdrawn.
lastPolledMu sync.Mutex
lastPolled map[string]polledIssues
// reactionPublisher is built lazily by scanPRReviews; an indirection
// only so tests can swap the delivery without the bus.
reactionPublisher ReactionPublisher
// Reactions is the review-reaction stream, drained into remediate runs each
// cycle. Nil disables it.
Reactions ReactionSource
// RunCredentials issues each task's run credential (see runCredential).
// Required whenever ConnectedStateStore.URL is set: the task is parked
// rather than handed the daemon's own administrative token.
RunCredentials RunCredentialIssuer
// ModelEgress gives native task containers model access without provider
// keys. Nil parks a task whose providers need a key.
ModelEgress ModelEgress
// TaskModels answers the model aliases a task's org resolves against.
// Nil runs every task on the instance aliases.
TaskModels TaskModels
// TaskInstructions answers the instance and organisation instructions a
// task's agent runs follow. Nil runs without them.
TaskInstructions TaskInstructions
// TaskRunReadyTimeout bounds how long runViaAgent retries a taskrun request
// while the new container has not subscribed yet. Zero uses
// defaultTaskRunReadyTimeout.
TaskRunReadyTimeout time.Duration
// TaskRunRetryBackoff is the delay between retry attempts within
// TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff.
TaskRunRetryBackoff time.Duration
// AgentStatus records agent versions for update verification. Nil disables
// it.
AgentStatus *AgentStatus
// ContainerPool manages Docker container lifecycle. Nil means autonomous
// execution is unavailable and process parks tasks; every runnable task gets
// a fresh container.
ContainerPool *container.Pool
// dispatcher bounds task execution globally across poll passes. Built once,
// on first use, so ResizeTaskDispatcher can apply a live
// containers.max_concurrency change without a restart; drainNATS submits
// each pass through it and leaves its tasks running.
dispatcherOnce sync.Once
dispatcher *taskDispatcher
// Storage is the pluggable storage backend for container mounts.
// A runnable task requires it; acquireTaskContainer parks on nil.
Storage storage.Backend
// IssueSources resolves, each poll cycle, every organisation connection
// to poll and the identity whose permit reaches it. Nil polls nothing.
IssueSources func(context.Context) []IssueSource
// OrgPrincipals is one acting principal per organisation, system org
// first. Nil dispatches the root identity's org alone.
OrgPrincipals func(context.Context) []access.Principal
// IdentityRepository is the durable lifecycle authority. Polling checks it
// every cycle so suspend/reactivate commands do not require a restart.
IdentityRepository identity.Repository
AgentAssignments org.AgentAssignments
// PackageAuthorities bounds the tools a package workflow's run is offered.
PackageAuthorities PackageAuthorities
// Playbooks are the loaded EDA playbooks, consulted before kind/label
// bindings when pinning a workflow. Nil means none.
Playbooks interface {
Dispatch(context.Context, playbook.DispatchInput) (playbook.Decision, bool)
Run(context.Context, storecontract.PlaybookDispatcher, *slog.Logger, playbook.DispatchInput) error
}
// PlaybookLedger is the at-most-once gate action playbooks run through.
// Nil means action playbooks do not run: they never fire unguarded.
PlaybookLedger storecontract.PlaybookDispatcher
// WorkflowDefinitions supplies the active database definitions. A task is
// pinned once before dispatch; retries reuse the task's stored YAML.
WorkflowDefinitions interface {
WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error)
}
// WithdrawTask declines a task whose issue a complete poll no longer
// returns. Nil leaves withdrawn issues' work running.
WithdrawTask func(ctx context.Context, taskID int64, reason string) error
// WorkflowEnablement supplies which workflows each org has disabled. A
// binding targeting a workflow its org disabled does not dispatch. Nil
// means every workflow is enabled.
WorkflowEnablement interface {
WorkflowEnablement(context.Context) (workflowtask.WorkflowEnablement, error)
}
// TaskLogs persists each task's log output and mirrors it to the dashboard.
// Nil disables task logging.
TaskLogs *logging.TaskRegistry
// running holds a cancel function for every task currently executing,
// so /stop can reach work already in flight. Its zero value is ready
// to use.
running runningTasks
// loopPasses maps each run loop ("cycle", "drain") to when its latest
// pass began, as Unix nanoseconds. StalestLoop reports the oldest, so one
// wedged loop isn't hidden by a healthy one.
loopPasses sync.Map
}
// IssueSource is one forge connection's repositories as the identity
// permitted to read them sees them.
type IssueSource struct {
Identity identity.IdentityID
Forge forge.Forge
// BotUser is the connection's external account, the assignee polled for.
BotUser string
// Dispatch and Label are the discovery rule the connection's org polls
// with. Empty means the daemon's own.
Dispatch config.Dispatch
Label string
Repos []config.Repo
}
func (d *Daemon) issueSources(ctx context.Context) []IssueSource {
if d.IssueSources == nil {
return nil
}
return d.IssueSources(ctx)
}
// emit publishes an observability event; safe on a nil bus.
func (d *Daemon) emit(ctx context.Context, e events.Event) {
if d.Bus != nil {
d.Bus.Publish(ctx, e)
}
}
// Startup runs crash recovery and access verification once.
func (d *Daemon) Startup(ctx context.Context) error {
if err := d.cleanupExpiredStorage(ctx); err != nil {
d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err)
}
n, err := d.Store.RecoverStale(ctx)
if err != nil {
return err
}
if n > 0 {
d.Log.InfoContext(ctx, "re-queued tasks left running by a previous daemon", "count", n)
}
d.sweepAccess(ctx)
return nil
}
// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and
// the run loop, and a forge client carries no request deadline of its own, so
// an unreachable host would otherwise hold boot open indefinitely.
const sweepTimeout = 2 * time.Minute
// A clone waits, holding its slot, until the work volume has minCloneSpace
// free, and parks as transient if it does not within diskWaitLimit. Clones
// share objects with the repository's mirror, so the floor covers a checkout
// and its build output, not a full clone.
const (
minCloneSpace = 1 << 30
diskCheckEvery = 15 * time.Second
diskWaitLimit = 10 * time.Minute
// cloneRetention is how long an untouched clone, such as a parked task's,
// is kept before a full work volume may reclaim it.
cloneRetention = 72 * time.Hour
)
// sweepAccess accepts pending invitations and verifies push access through
// every organisation connection, each in its own goroutine.
func (d *Daemon) sweepAccess(ctx context.Context) {
sweep := func(source IssueSource) {
log := d.Log.With("identity", source.Identity)
ctx, cancel := context.WithTimeout(ctx, sweepTimeout)
defer cancel()
if err := source.Forge.AcceptInvitations(ctx, repoRefs(source.Repos)); err != nil {
log.WarnContext(ctx, "invitation sweep failed", "err", err)
}
for _, r := range source.Repos {
if err := source.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil {
// A sweep cut short by its own context is not a repo failure: name
// the budget when it expired, stay quiet on a shutdown.
switch {
case errors.Is(ctx.Err(), context.DeadlineExceeded):
// Every later call would return the same error, so say it once.
log.WarnContext(ctx, "push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
return
case ctx.Err() != nil:
return
}
log.WarnContext(ctx, "repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
}
}
}
var wg sync.WaitGroup
for _, source := range d.issueSources(ctx) {
wg.Go(func() { sweep(source) })
}
wg.Wait()
}
// repoRefs is a source's configured repositories as the allowlist the forge
// accepts invitations against.
func repoRefs(repos []config.Repo) []forge.RepoRef {
out := make([]forge.RepoRef, 0, len(repos))
for _, r := range repos {
out = append(out, forge.RepoRef{Owner: r.Owner, Repo: r.Name})
}
return out
}
// Run polls until the context ends.
func (d *Daemon) Run(ctx context.Context) error {
// A pass no longer waits for the tasks it submitted, so the run does, once
// the ctx is cancelled.
defer d.WaitForTasks()
d.watchCallWaits(ctx)
if d.Tasks != nil {
drained := make(chan struct{})
defer func() { <-drained }()
go func() {
defer close(drained)
d.drainLoop(ctx)
}()
}
interval := d.Cfg.Get().PollInterval.Std()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
if d.Tasks == nil {
d.dispatchBindings(ctx)
}
d.Cycle(ctx)
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
// PollInterval may have been reloaded; reset the ticker so a
// change takes effect on the next cycle.
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
interval = iv
ticker.Reset(iv)
}
}
}
}
// drainLoop is the only task drainer and capture dispatcher while Run is
// active. It runs apart from the poll tick so a captured event becomes a
// running task within about a second regardless of poll_interval or forge
// latency. Submit never blocks, so a full dispatcher
// just leaves tasks waiting on slots and the loop stays paced by the ticker.
func (d *Daemon) drainLoop(ctx context.Context) {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
d.markPass("drain")
d.dispatchBindings(ctx)
d.drainNATS(ctx)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
// pollIssuesWithConfig returns the repo's issues the dispatch rule makes
// work, using the given forge client and dispatch config. complete is false when any part of the poll failed or was skipped, so
// the result must not be read as the whole eligible set.
func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) (issues []forge.Issue, complete bool) {
labelled := func() ([]forge.Issue, bool) {
// A label trigger needs a non-empty label or it would match every open issue.
if cfg.Label == "" {
d.Log.ErrorContext(ctx, "label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName())
return nil, false
}
issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
if err != nil {
d.Log.ErrorContext(ctx, "label poll failed", "repo", repo.FullName(), "err", err)
return nil, false
}
return issues, true
}
assigned := func() ([]forge.Issue, bool) {
if cfg.BotUser == "" {
d.Log.ErrorContext(ctx, "assignee trigger needs the connection's account; refusing to poll", "repo", repo.FullName())
return nil, false
}
issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
if err != nil {
d.Log.ErrorContext(ctx, "assigned poll failed", "repo", repo.FullName(), "err", err)
return nil, false
}
return issues, true
}
switch cfg.Dispatch.Trigger {
case "label":
return labelled()
case "either":
byAssignee, assignedOK := assigned()
byLabel, labelledOK := labelled()
seen := map[int]bool{}
for _, is := range byAssignee {
seen[is.Number] = true
}
issues = byAssignee
for _, is := range byLabel {
if !seen[is.Number] {
issues = append(issues, is)
}
}
return issues, assignedOK && labelledOK
default: // "assignee" (default)
return assigned()
}
}
// RunOnce runs one cycle and drains until no task is in flight. A run waiting
// on a callee stays in flight while the callee sits queued, so the queue is
// drained again until every task, callees included, has returned.
func (d *Daemon) RunOnce(ctx context.Context) {
defer d.WaitForTasks()
d.watchCallWaits(ctx)
// One pass has no scheduler, so it runs the maintenance jobs itself.
for _, job := range d.BuiltinJobs() {
if err := job.Run(ctx); err != nil {
d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err)
}
}
d.dispatchBindings(ctx)
d.Cycle(ctx)
if d.Tasks == nil {
return
}
d.drainNATS(ctx)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for d.taskDispatcher().inFlight() > 0 {
select {
case <-ctx.Done():
return
case <-ticker.C:
d.drainNATS(ctx)
}
}
}
// Cycle is one poll pass: turn review reactions into tasks; maintenance runs
// as built-in schedules. Captured events are dispatched by the caller
// (drainLoop in Run, once in RunOnce) so they start within about a second.
func (d *Daemon) Cycle(ctx context.Context) {
d.markPass("cycle")
d.drainReactions(ctx)
d.retryTransientParks(ctx)
}
// retryTransientParks requeues tasks parked for an environmental cause once
// their backoff has passed. The store parks a task needs_decision, which
// raises the alert, when its retries run out, so this never loops on one.
func (d *Daemon) retryTransientParks(ctx context.Context) {
if d.Store == nil {
return
}
now := time.Now()
var after storecontract.TaskCursor
for {
page, err := d.Store.TasksPage(ctx, storecontract.TaskPage{Statuses: []string{workflow.StatusParked}, After: after, Limit: 200})
if err != nil {
d.Log.WarnContext(ctx, "list parked tasks for retry failed", "err", err)
return
}
for _, t := range page {
if taskstate.TransientRetryDue(t.ParkClass, t.RetryCount, t.UpdatedAt, now) {
d.retryTransientPark(ctx, t.ID)
}
}
if len(page) < 200 {
return
}
last := page[len(page)-1]
after = storecontract.TaskCursor{UpdatedAt: last.UpdatedAt, ID: last.ID}
}
}
// retryTransientPark requeues one task on its stored retry mode, which the
// list projection omits: a remediation must keep continuing its branch.
func (d *Daemon) retryTransientPark(ctx context.Context, id int64) {
t, err := d.Store.TaskByID(ctx, id)
if err != nil {
d.Log.WarnContext(ctx, "load task for transient retry failed", "task", id, "err", err)
return
}
if err := d.Store.RetryTask(ctx, id, workflow.StatusParked, "", t.RetryMode, workflowtask.Resume{}, false); err != nil {
if !errors.Is(err, storecontract.ErrStaleTransition) {
d.Log.WarnContext(ctx, "transient park retry failed", "task", id, "err", err)
}
return
}
d.emit(ctx, events.Event{
Kind: events.KindTaskRetried, TaskID: id, Workflow: t.Workflow,
Repo: strings.Trim(t.Owner+"/"+t.Repo, "/"), Issue: t.IssueNumber,
Detail: "retried automatically after a transient park",
Data: map[string]any{"retry_count": t.RetryCount + 1, "previous_reason": t.ParkReason},
})
}
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
if ttl <= 0 || d.Storage == nil {
return nil
}
n, err := d.Storage.CleanupExpired(ctx, ttl)
if n > 0 {
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
}
if err != nil {
return fmt.Errorf("persistent volume cleanup: %w", err)
}
return nil
}
// bindingDispatchBatchLimit caps how many undispatched captures a single
// dispatchBindings pass will fetch from the store. The cap is defensive
// (the captures table is also pruned by retention/maxEvents) and bounds a
// single cycle's worst-case latency on a backlog.
const bindingDispatchBatchLimit = 100
// dispatchBindings offers each identified capture to every armed binding on
// its source. Nil Bindings, BindingDispatcher or BindingTaskCreator disables
// it.
func (d *Daemon) dispatchBindings(ctx context.Context) {
if d.Bindings == nil || d.BindingDispatcher == nil || d.BindingTaskCreator == nil {
return
}
if d.Principals == nil {
d.dispatchOrgBindings(ctx)
return
}
var principals []access.Principal
if d.OrgPrincipals != nil {
principals = d.OrgPrincipals(ctx)
} else {
principal, err := d.Principals.PrincipalFor(ctx, identity.SystemID)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: principal unavailable", "identity", identity.SystemID, "err", err)
return
}
principals = []access.Principal{principal}
}
seen := make(map[org.OrgID]bool)
for _, principal := range principals {
if seen[principal.Org] {
continue
}
seen[principal.Org] = true
d.dispatchOrgBindings(org.WithOrg(access.WithPrincipal(ctx, principal), principal.Org))
}
}
func (d *Daemon) dispatchOrgBindings(ctx context.Context) {
// Gather the armed bindings whose workflow is enabled first, so the
// oldest-first captures query is not held by captures that only a binding
// on a disabled workflow would take.
bindings, err := d.Bindings.ListBindings(ctx)
if err != nil {
d.Log.WarnContext(ctx, "list bindings failed", "error", err)
return
}
enablement, err := d.workflowEnablement(ctx)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: workflow enablement unavailable", "error", err)
return
}
enabled := make([]string, 0, len(bindings))
armed := make([]binding.Binding, 0, len(bindings))
for _, b := range bindings {
if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) {
enabled = append(enabled, b.ID)
armed = append(armed, b)
}
}
if len(enabled) == 0 {
return
}
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit)
if err != nil {
d.Log.WarnContext(ctx, "list undispatched captures failed", "error", err)
return
}
workflows, err := d.activeWorkflows(ctx)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: workflow definitions unavailable", "error", err)
return
}
for _, c := range captures {
d.dispatchCapture(ctx, c, armed, workflows)
}
}
// dispatchCapture offers one capture to every armed binding that matches it.
// armed holds only the org's armed bindings on enabled workflows, so a
// disabled workflow's capture waits rather than recording a failure every pass.
func (d *Daemon) dispatchCapture(ctx context.Context, c storecontract.CapturedEvent, armed []binding.Binding, workflows workflow.WorkflowDefinitionCollection) {
for _, b := range armed {
if b.Matcher.Matches(c.Source, c.Dispatchable()) {
d.dispatchOneBinding(ctx, b, c, workflows)
}
}
}
// bindingMapping returns b's mapping, or nil after logging why it is
// unavailable. A mapping that no longer exists is a terminal outcome.
func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent) *mapping.Mapping {
if d.Mappings == nil {
d.Log.WarnContext(ctx, "binding dispatch: mapping store unavailable", "binding", b.ID)
return nil
}
m, err := d.Mappings.GetMapping(ctx, b.MappingID)
if err != nil {
d.Log.WarnContext(ctx, "binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err)
return nil
}
if m == nil {
d.recordDispatchFailure(ctx, b, c, fmt.Sprintf("mapping %q does not exist", b.MappingID), nil)
}
return m
}
// dispatchOneBinding offers one capture to one binding: the mapping must
// match the capture's event type and resolve required fields, the filter
// must admit the values, and the workflow must accept the inputs. The
// ledger claim and the task commit together, so each binding fires exactly
// once per delivery.
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) {
m := d.bindingMapping(ctx, b, c)
if m == nil || m.EventTypeID != c.EventType {
return
}
values, failures := mapping.Resolve(m.Fields, []byte(c.Body))
if hasBlockingFailure(m.Fields, failures) {
d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures})
return
}
if d.MappingMatches != nil {
if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil {
d.Log.WarnContext(ctx, "binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err)
}
}
filter, err := binding.CompileFilter(b.Filter, m.Fields)
if err != nil {
d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()})
return
}
if admitted, _ := filter.Admits(values); !admitted {
d.recordEvaluated(ctx, b, c, "filtered")
return
}
target, reason, ok := d.resolveBindingTarget(b, values, workflows)
if !ok {
d.recordDispatchFailure(ctx, b, c, reason, nil)
return
}
// The chain decides dispatch: the binding's principal may `run` this
// workflow in its workspace, and the event's signature result and address
// travel with the request. A denial is terminal; an unavailable principal
// or policy chain leaves the capture listed for the next cycle.
if !d.authorizeDispatch(ctx, b, c, target) {
return
}
d.claimAndEnqueue(ctx, b, c, target, values)
}
// claimAndEnqueue asks the State Store to claim the dispatch and enqueue its
// task as one unit. A failure leaves the delivery undispatched, so the next
// cycle offers it again; a delivery already dispatched is refused.
func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) {
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
body := renderBindingBody(values, c)
if target.declared {
body = fmt.Sprintf("Started by binding %q from capture %q; the workflow's inputs travel with the task.", b.Name, c.ID)
}
taskIdentity := string(identity.SystemID)
if principal, ok := access.PrincipalFromContext(ctx); ok {
taskIdentity = string(principal.IdentityID)
}
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs)
if err != nil {
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
d.Log.WarnContext(ctx, "binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
}
return
}
d.markBindingStart(ctx, task.ID, b, c)
}
// authorizeDispatch checks the binding's principal, its org's dispatching
// identity, may start the workflow. A denial is recorded and reported as a
// binding_dispatch_failure. The run itself acts as the workflow's declared
// identity, which adoptWorkflowIdentity checks when the task is processed.
func (d *Daemon) authorizeDispatch(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget) bool {
if d.Access == nil || d.Principals == nil {
return true
}
principal, ok := access.PrincipalFromContext(ctx)
if !ok {
d.Log.WarnContext(ctx, "binding dispatch: no principal; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID)
return false
}
resource := access.Resource{
Kind: access.KindWorkflow, ID: b.Workflow, Org: principal.Org, Workspace: b.WorkspaceID,
}
contextValue := access.Context{Signature: signatureOf(c), Addr: c.RemoteAddr}
decision := d.Access.Authorize(principal, access.ActionRun, resource, contextValue)
if decision.Allowed {
return true
}
if errors.Is(decision.Err, access.ErrChainUnavailable) {
// Nothing was evaluated. Recording it would mark the pair evaluated for
// good, so a State Store blip would drop the capture instead of leaving
// it for the next cycle.
d.Log.WarnContext(ctx, "binding dispatch: access chain unavailable; dispatch left for the next cycle", "binding", b.ID, "capture", c.ID, "err", decision.Err)
return false
}
if d.Denials != nil {
denial := access.Denial{
Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun,
Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level,
Policies: decision.Policies,
}
if err := d.Denials.RecordDenial(ctx, denial); err != nil {
d.Log.WarnContext(ctx, "binding dispatch: record denial", "binding", b.ID, "error", err)
}
}
d.recordDispatchFailure(ctx, b, c, "access denied", map[string]any{
"level": string(decision.Level), "policies": decision.Policies,
})
return false
}
// signatureOf names the event's signature result as the dispatch context
// carries it: a signed event and an approved unsigned event are the two
// dispatchable shapes (storecontract.CapturedEvent.Dispatchable).
func signatureOf(c storecontract.CapturedEvent) string {
switch {
case c.Authenticated:
return "valid"
case c.Unsigned:
return "unsigned"
}
return "unverified"
}
// markBindingStart records on a task's timeline the binding and captured
// event that started it.
func (d *Daemon) markBindingStart(ctx context.Context, taskID int64, b binding.Binding, c storecontract.CapturedEvent) {
if _, err := d.Store.InsertEvent(ctx, events.Event{
TaskID: taskID,
Kind: events.KindBindingStarted,
Detail: fmt.Sprintf("started by binding %q from source %q", b.Name, c.Source),
Data: map[string]any{
"binding_id": b.ID, "binding_name": b.Name, "binding_version": b.Version,
"capture_id": c.ID, "source": c.Source, "unsigned": c.Unsigned,
},
}); err != nil {
d.Log.WarnContext(ctx, "binding dispatch: start marker", "task", taskID, "error", err)
}
}
// recordDispatchFailure records why a binding did not dispatch a capture and
// marks the pair evaluated, so the capture is not offered to it again.
func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) {
d.recordEvaluated(ctx, b, c, reason)
data := map[string]any{
"binding_id": b.ID,
"binding_version": b.Version,
"capture_id": c.ID,
}
maps.Copy(data, extra)
_, _ = d.Store.InsertEvent(ctx, events.Event{
Kind: "binding_dispatch_failure",
Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason),
Data: data,
})
}
// recordEvaluated writes the ledger row for an evaluation that started no
// task. Only terminal outcomes call it; transient ones leave the capture
// listed for the next cycle.
func (d *Daemon) recordEvaluated(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string) {
err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, reason)
if err != nil && !errors.Is(err, storecontract.ErrAlreadyDispatched) {
d.Log.WarnContext(ctx, "binding dispatch: record evaluation", "binding", b.ID, "capture", c.ID, "error", err)
}
}
// hasBlockingFailure reports whether any failure is on a required field.
func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool {
if len(failures) == 0 {
return false
}
required := make(map[string]bool, len(fields))
for _, f := range fields {
if f.Required {
required[f.Name] = true
}
}
for _, fail := range failures {
if required[fail.FieldName] {
return true
}
}
return false
}
// renderBindingBody builds the task body from the values a mapping
// resolved, one "key=value" pair per line. Empty maps still produce
// a body that records the source capture id so the workflow has
// provenance to hand back to the operator.
func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string {
if len(values) == 0 {
return fmt.Sprintf("(no fields resolved from capture %q)", c.ID)
}
parts := make([]string, 0, len(values))
for k, v := range values {
parts = append(parts, fmt.Sprintf("%s=%v", k, v))
}
return strings.Join(parts, "\n")
}
// resolveBindingRepo returns the binding's pinned owner/repo, or the only
// configured repo. With zero or several repos and no pin it refuses.
func (d *Daemon) resolveBindingRepo(b binding.Binding) (owner, repo, reason string) {
if b.Owner != "" && b.Repo != "" {
return b.Owner, b.Repo, ""
}
repos := d.Cfg.Get().Repos
if len(repos) == 1 {
return repos[0].Owner, repos[0].Name, ""
}
return "", "", fmt.Sprintf("cannot pick a repository: %d are configured; pin the binding to one", len(repos))
}
// drainReactions turns available review reactions into queued remediate
// runs. It runs before the task drain so a reaction queued in this pass can
// be claimed by it; the consumer is built lazily once, so its drop counter
// spans the process lifetime.
func (d *Daemon) drainReactions(ctx context.Context) {
if d.Reactions == nil || d.Store == nil {
return
}
if d.reactionConsumer == nil {
d.reactionConsumer = newReactionConsumer(
// The store's owned-PR lookup is the authorization boundary: a
// reaction only ever causes work against a PR archie opened and
// still owns (pr-review-remediation.md decision 3).
d.Store,
d.Store,
d.Store,
d.botUserForTask,
d.Log,
)
}
if _, err := d.reactionConsumer.drain(ctx, d.Reactions, maxReactionsPerCycle); err != nil {
d.Log.ErrorContext(ctx, "reaction drain failed", "err", err)
}
}
// botUserForTask resolves the bot login that owns task, so the reaction
// consumer can exclude Archie's own comments: the global forge bot user for
// root tasks.
func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
if !d.rootTask(task) {
return ""
}
return d.Cfg.Get().BotUser
}
// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for
// requeued tasks (waiting_approval approval, retry-parked) that didn't come
// through a NATS publish. It returns once the stream and the claim fallback
// are both empty: in-flight tasks keep running, so a task waiting on a callee
// does not hold up the pass that would claim it. Shutdown waits for them
// through WaitForTasks.
func (d *Daemon) drainNATS(ctx context.Context) {
dispatcher := d.taskDispatcher()
// A full dispatcher leaves work in the stream and the store, not parked
// on slots where it already reads as running.
for ctx.Err() == nil && dispatcher.hasRoom() {
msg, err := d.Tasks.Fetch(ctx)
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
d.Log.ErrorContext(ctx, "nats fetch failed", "err", err)
break
}
if err == nil {
d.submitNATSTask(ctx, dispatcher, msg)
d.markPass("drain")
continue
}
// ErrNoMessage means the stream is drained. That is precisely when
// the SQLite fallback must run: requeued tasks (waiting_approval
// approval, retry-parked) never went through a NATS publish, so
// nothing will ever arrive on the stream to trigger them.
d.claimRequeued(ctx, dispatcher)
return
}
}
// claimRequeued submits tasks from the store's claim fallback while the
// dispatcher has room; the rest stay unclaimed for a later pass. It does
// not go back to the stream between claims: each empty fetch waits a full poll
// timeout, so a backlog left by a restart would outlast the systemd watchdog.
func (d *Daemon) claimRequeued(ctx context.Context, dispatcher *taskDispatcher) {
for ctx.Err() == nil && dispatcher.hasRoom() {
task, err := d.Store.ClaimNext(ctx)
if err != nil {
d.Log.ErrorContext(ctx, "sqlite claim failed", "err", err)
return
}
if task == nil {
return
}
dispatcher.Submit(ctx, task, d.process)
d.markPass("drain")
}
}
// taskDispatcher returns the daemon's long-lived global dispatcher, building
// it from the running config on first use. It outlives a single drain pass so
// ResizeTaskDispatcher can resize the running dispatcher.
func (d *Daemon) taskDispatcher() *taskDispatcher {
d.dispatcherOnce.Do(func() {
d.dispatcher = newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask)
})
return d.dispatcher
}
// WaitForTasks blocks until every submitted task's process has returned. A
// drain pass no longer waits for its own tasks, so shutdown is where in-flight
// work is waited for.
func (d *Daemon) WaitForTasks() {
d.taskDispatcher().Wait()
}
// ResizeTaskDispatcher applies a new containers.max_concurrency to the
// running global task dispatcher without a restart. Running tasks keep their
// slots; a lowered limit is honoured as they finish, and a raised limit
// admits queued work immediately.
func (d *Daemon) ResizeTaskDispatcher(maxConcurrency int) {
d.taskDispatcher().SetMaxConcurrency(maxConcurrency)
}
// submitNATSTask decodes a fetched message and queues it for processing. A
// message that cannot be decoded is acked rather than redelivered: it will
// never decode on a retry, so leaving it unacked would block the queue.
func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) {
envelope, err := workintake.DecodeTask(msg.Data())
if err != nil {
d.Log.ErrorContext(ctx, "nats decode failed", "subject", msg.Subject(), "err", err)
_ = msg.Ack()
return
}
task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task, hold *taskHold) {
d.processNATSTask(ctx, msg, hold)
})
}
// taskDispatcher limits concurrent tasks globally and runs one task per repo
// at a time, in submit order, unless the repo allows concurrency.
type taskDispatcher struct {
allowConcurrent func(task *workflow.Task) bool
mu sync.Mutex
limit int
active int
slotsFree *sync.Cond
repoTail map[string]chan struct{}
pending int
// waiting counts submitted tasks that hold no slot yet, so a drain pass
// stops claiming once queued work alone fills the limit.
waiting int
// holds maps a task id to the capacity its run holds, so a run waiting on
// a callee can give that capacity back and take it again by task id.
holds map[int64]*taskHold
}
// taskHold is one submitted task's claim on execution capacity: its slot under
// the global limit, and its place at the tail of its repo's chain, which the
// next task for that repo is blocked on. A run that waits on a callee gives
// both back so the callee can run, and takes them again when the wait ends.
type taskHold struct {
dispatcher *taskDispatcher
repo string
// chain is true when the repo runs one task at a time, so the run holds a
// place in that repo's chain.
chain bool
// baton is the run's place in its repo's chain, closed when it gives that
// place up; nil while it holds none.
baton chan struct{}
// slot is true while the run counts against the global limit.
slot bool
// waits counts the callees the run is waiting on: an open call gives capacity
// back and the last closed call takes it again.
waits int
// finished is true once the run's process returned.
finished bool
// id is the task this hold belongs to, named by the run once it knows the
// task's id.
id int64
attempt int
}
// name records which task's capacity this hold is, so a wait on a callee can
// give it back by task id. Zero names nothing: a task arriving on the NATS
// stream has no id until the store claims its row inside the run.
func (h *taskHold) name(id int64, attempt int) {
if id == 0 {
return
}
h.dispatcher.mu.Lock()
h.id = id
h.attempt = attempt
h.dispatcher.holds[id] = h
h.dispatcher.mu.Unlock()
}
// SetMaxConcurrency accepts a new global limit without a restart. Running
// tasks keep their slots; a lowered limit is enforced as they finish, and a
// raised limit admits queued work immediately.
func (d *taskDispatcher) SetMaxConcurrency(maxConcurrency int) {
d.mu.Lock()
d.limit = maxConcurrency
d.mu.Unlock()
d.slotsFree.Broadcast()
}
func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher {
if allowConcurrent == nil {
allowConcurrent = func(*workflow.Task) bool { return false }
}
d := &taskDispatcher{
limit: maxConcurrency,
allowConcurrent: allowConcurrent,
repoTail: make(map[string]chan struct{}),
holds: make(map[int64]*taskHold),
}
d.slotsFree = sync.NewCond(&d.mu)
return d
}
// joinRepoLocked appends baton to the tail of repo's chain and returns the
// baton ahead of it, which the joining task is blocked on. Callers hold d.mu.
func (d *taskDispatcher) joinRepoLocked(repo string, baton chan struct{}) chan struct{} {
previous := d.repoTail[repo]
d.repoTail[repo] = baton
return previous
}
// takeSlotLocked blocks until the global limit has room, then counts the run
// in. limit <= 0 means unlimited. The limit is re-read on every wake, so
// SetMaxConcurrency reaches waiting tasks without recreating the dispatcher.
// Callers hold d.mu.
func (d *taskDispatcher) takeSlotLocked() {
for d.limit > 0 && d.active >= d.limit {
d.slotsFree.Wait()
}
d.active++
}
// giveBackLocked gives back everything the run holds: its slot under the
// global limit, and its place in its repo's chain, which releases the task
// queued behind it. Callers hold d.mu.
func (d *taskDispatcher) giveBackLocked(hold *taskHold) {
if hold.slot {
d.active--
hold.slot = false
}
if hold.baton == nil {
return
}
close(hold.baton)
if d.repoTail[hold.repo] == hold.baton {
delete(d.repoTail, hold.repo)
}
hold.baton = nil
}
// finishRun retires a returned run: it holds no capacity any more, and no
// Resume may take any for it. Whatever an in-flight resume takes later is
// given back by reacquire, which sees finished.
func (d *taskDispatcher) finishRun(hold *taskHold) {
d.mu.Lock()
hold.finished = true
if hold.id != 0 && d.holds[hold.id] == hold {
delete(d.holds, hold.id)
}
d.giveBackLocked(hold)
d.pending--
d.mu.Unlock()
d.slotsFree.Broadcast()
}
// setCallWaits applies a durable snapshot without accumulating duplicate events.
func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
d.mu.Lock()
if d.holds[hold.id] != hold || hold.finished {
d.mu.Unlock()
return
}
previous := hold.waits
hold.waits = waits
if waits > 0 {
d.giveBackLocked(hold)
}
d.mu.Unlock()
if waits > 0 {
d.slotsFree.Broadcast()
} else if previous > 0 {
// Reacquiring may block behind another task; the caller keeps running.
go d.reacquire(hold)
}
}
// reacquire returns a resumed run's capacity: its place in its repo's chain
// and a slot under the global limit. The run keeps executing meanwhile -- a
// wait on a callee is not a pause. A hold records one slot and one place, so a
// re-acquire queued while another is still waiting gives back whichever of the
// two it finds already recorded instead of taking a second.
func (d *taskDispatcher) reacquire(hold *taskHold) {
d.mu.Lock()
if hold.finished || hold.waits > 0 {
// The run returned, or started waiting again, after the reconciliation that
// queued this: it needs no capacity.
d.mu.Unlock()
return
}
var previous chan struct{}
// A place another re-acquire already queued is the run's one place; joining
// behind it would queue a second, which the task behind it would wait on
// forever.
if hold.chain && hold.baton == nil {
hold.baton = make(chan struct{})
previous = d.joinRepoLocked(hold.repo, hold.baton)
}
d.mu.Unlock()
if previous != nil {
<-previous
}
d.mu.Lock()
d.takeSlotLocked()
if hold.slot {
// Another re-acquire took the run's one slot while this waited: give
// back the slot this one just took. The place is the other's, so it is
// left alone.
d.active--
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
hold.slot = true
if hold.finished || hold.waits > 0 {
// The run ended, or started waiting again, while this was queued for
// capacity: give back what it took, so no slot or place in the repo
// chain leaks.
d.giveBackLocked(hold)
d.mu.Unlock()
d.slotsFree.Broadcast()
return
}
d.mu.Unlock()
}
// Submit queues a task's process behind the global limit and, unless the repo
// allows concurrency, behind the task already holding that repo. process is
// handed the run's capacity so it can name it once it knows the task's id and
// give it back while it waits on a callee.
func (d *taskDispatcher) Submit(
ctx context.Context,
task *workflow.Task,
process func(context.Context, *workflow.Task, *taskHold),
) {
repo := task.Owner + "/" + task.Repo
hold := &taskHold{dispatcher: d, repo: repo}
var previous chan struct{}
if !d.allowConcurrent(task) {
hold.chain = true
hold.baton = make(chan struct{})
d.mu.Lock()
previous = d.joinRepoLocked(repo, hold.baton)
d.mu.Unlock()
}
d.mu.Lock()
d.pending++
d.waiting++
d.mu.Unlock()
go func() {
defer d.finishRun(hold)
if previous != nil {
<-previous
}
d.mu.Lock()
d.takeSlotLocked()
d.waiting--
hold.slot = true
d.mu.Unlock()
process(ctx, task, hold)
}()
}
// hasRoom reports whether a newly submitted task could take a slot without
// queueing behind work already submitted. A run waiting on a callee holds no
// slot, so its callee still finds room.
func (d *taskDispatcher) hasRoom() bool {
d.mu.Lock()
defer d.mu.Unlock()
return d.limit <= 0 || d.active+d.waiting < d.limit
}
// inFlight is how many submitted tasks have not returned yet.
func (d *taskDispatcher) inFlight() int {
d.mu.Lock()
defer d.mu.Unlock()
return d.pending
}
func (d *taskDispatcher) Wait() {
d.mu.Lock()
defer d.mu.Unlock()
for d.pending > 0 {
d.slotsFree.Wait()
}
}
// poll discovers work through every organisation connection, resolved
// afresh each cycle so a rotated credential or a new connection applies to
// the next poll.
func (d *Daemon) poll(ctx context.Context) error {
var failed []string
room := d.Cfg.Get().IntakePerPass
for _, source := range d.issueSources(ctx) {
if !d.identityActive(ctx, source.Identity) {
continue
}
cfg := d.Cfg.Get()
cfg.BotUser = source.BotUser
if source.Dispatch.Trigger != "" {
cfg.Dispatch, cfg.Label = source.Dispatch, source.Label
}
tag := string(source.Identity)
for _, repo := range source.Repos {
issues, complete := d.pollIssuesWithConfig(ctx, source.Forge, cfg, repo)
if complete {
d.withdrawUnpolled(ctx, repo, tag, dispatchRule(cfg), issues)
} else {
failed = append(failed, repo.FullName())
}
if d.Tasks != nil {
d.publishDiscovered(ctx, source.Forge, cfg, repo, issues, tag, &room)
}
}
}
return passFailed("issue poll", failed)
}
// passFailed reports a maintenance pass whose items failed; each item's cause
// is already logged where it happened.
func passFailed(pass string, failed []string) error {
if len(failed) == 0 {
return nil
}
return fmt.Errorf("%s failed for %d item(s): %s", pass, len(failed), strings.Join(failed, ", "))
}
// polledIssues is one repo's eligible issues under one dispatch rule.
type polledIssues struct {
rule string
eligible map[int]bool
}
// rememberPoll records a repo's latest eligible set and returns the previous
// one, reporting whether it was taken under the same rule.
func (d *Daemon) rememberPoll(key string, current polledIssues) (polledIssues, bool) {
d.lastPolledMu.Lock()
defer d.lastPolledMu.Unlock()
previous, seen := d.lastPolled[key]
if d.lastPolled == nil {
d.lastPolled = map[string]polledIssues{}
}
d.lastPolled[key] = current
return previous, seen && previous.rule == current.rule
}
// dispatchRule identifies the rule a poll selects issues by.
func dispatchRule(cfg config.Config) string {
return cfg.Dispatch.Trigger + "\x00" + cfg.Label + "\x00" + cfg.BotUser
}
// withdrawUnpolled declines the queued or running work of an identity's
// issues in repo that a complete poll no longer returns: the issue was
// closed, unlabelled or unassigned.
func (d *Daemon) withdrawUnpolled(ctx context.Context, repo config.Repo, identity, rule string, polled []forge.Issue) {
if d.WithdrawTask == nil {
return
}
eligible := make(map[int]bool, len(polled))
for _, is := range polled {
eligible[is.Number] = true
}
// Only an issue the same rule found eligible last time and no longer
// finds was withdrawn. A changed label, trigger or bot user, or the first
// poll after a restart, only records the new set: tasks picked up under
// the old rule were not withdrawn by anyone.
previous, sameRule := d.rememberPoll(identity+"\x00"+repo.FullName(), polledIssues{rule: rule, eligible: eligible})
if !sameRule {
return
}
active, err := d.Store.TasksPage(ctx, storecontract.TaskPage{
Statuses: []string{taskstate.Queued, taskstate.Running}, Limit: 500,
})
if err != nil {
d.Log.WarnContext(ctx, "withdrawal check: active tasks unavailable", "repo", repo.FullName(), "err", err)
return
}
for _, t := range active {
if t.Owner != repo.Owner || t.Repo != repo.Name || t.Identity != identity ||
!t.IsForgeBacked() || t.PRNumber > 0 || eligible[t.IssueNumber] || !previous.eligible[t.IssueNumber] {
continue
}
if err := d.WithdrawTask(ctx, t.ID, "the issue is no longer assigned, labelled or open"); err != nil {
d.Log.WarnContext(ctx, "withdraw task", "task", t.ID, "err", err)
}
}
}
// publishDiscovered publishes a repo's discovered issues until the pass has
// created its cap of tasks. room is what is left of the cap; the issues left
// over are found again by the next poll, because an issue with a task is
// skipped and one without is not. A cap of zero or less is no cap.
func (d *Daemon) publishDiscovered(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, issues []forge.Issue, identity string, room *int) {
capped := *room > 0
for _, is := range issues {
if capped && *room == 0 {
d.Log.InfoContext(ctx, "intake cap reached; remaining issues wait for the next poll", "repo", repo.FullName())
return
}
if d.pollNATS(ctx, fg, cfg, repo, is, strings.Join(is.Labels, ","), identity) && capped {
*room--
}
}
}
// pollNATS publishes a discovered issue to NATS and reports whether it created
// a task.
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) bool {
// Read-only existence check prevents rediscovering an existing task.
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
if err != nil {
d.Log.ErrorContext(ctx, "task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return false
}
if existing != nil {
return false
}
// Classify and encode here: the envelope, its routing kind and its
// subject belong to the work-intake domain, not to the transport.
parsed := workintake.SplitLabels(labels)
task := workintake.TaskEnvelope{
Owner: repo.Owner,
Repo: repo.Name,
Number: is.Number,
Title: is.Title,
Body: is.Body,
Labels: parsed,
Identity: identity,
Kind: workintake.KindForLabels(parsed),
}
if err := d.PublishTask(ctx, task); err != nil {
d.Log.ErrorContext(ctx, "task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
return false
}
d.acknowledge(ctx, fg, cfg, repo, is)
return true
}
// PublishTask validates a task envelope, stamps the producing identity's
// org and publishes it. The idempotency key dedups rediscovered issues.
func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
if d.Tasks == nil {
return fmt.Errorf("publish task %s: no task bus configured", task.Ref())
}
if err := task.Kind.Validate(); err != nil {
return fmt.Errorf("publish task %s: %w", task.Ref(), err)
}
if task.Identity == "" {
return fmt.Errorf("publish task %s: no identity", task.Ref())
}
resolved, err := d.identityOrg(ctx, identity.IdentityID(task.Identity))
if err != nil {
return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err)
}
task.Org = resolved
payload, err := task.Encode()
if err != nil {
return err
}
return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
}
// PublishReaction stamps the org on a review reaction, defaulting to the
// root identity's, and publishes it.
func (d *Daemon) PublishReaction(ctx context.Context, reaction workintake.ReviewCommentEnvelope) error {
if d.Tasks == nil {
return fmt.Errorf("publish reaction %s: no task bus configured", reaction.Ref())
}
if reaction.Org == "" {
resolved, err := d.identityOrg(ctx, identity.SystemID)
if err != nil {
return fmt.Errorf("publish reaction %s: resolve org: %w", reaction.Ref(), err)
}
reaction.Org = resolved
}
payload, err := reaction.Encode()
if err != nil {
return err
}
return d.Tasks.PublishUnique(ctx, reaction.Subject(), reaction.IdempotencyKey(), payload)
}
// identityOrg resolves an identity's org. No PrincipalSource means the
// default org; a store error is returned.
func (d *Daemon) identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) {
if d.Principals == nil {
return org.DefaultOrgID, nil
}
principal, err := d.Principals.PrincipalFor(ctx, id)
if err != nil {
return "", err
}
return principal.Org, nil
}
// acknowledge posts the pickup reaction
// using fg -- the forge client that owns repo (identity-scoped or root) --
// and the ack reaction from cfg, the dispatch config that owns this poll.
func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) {
d.Log.InfoContext(ctx, "issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title)
if ack := cfg.Dispatch.AckReaction; ack != "" {
if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil {
d.Log.WarnContext(ctx, "ack reaction failed", "issue", is.Number, "err", err)
}
}
}
// emitTaskQueued records the queued task under the identity that polled it,
// which is both the actor and the authority the task was created with.
func (d *Daemon) emitTaskQueued(ctx context.Context, task *workflow.Task) {
e := events.Event{
Kind: events.KindTaskQueued, TaskID: task.ID, Repo: task.Owner + "/" + task.Repo,
Issue: task.IssueNumber, Detail: task.Title, Attempt: task.Attempt,
ActorID: task.Identity, PrincipalID: task.Identity,
}
if d.IdentityRepository != nil {
if value, err := d.IdentityRepository.Get(ctx, identity.IdentityID(task.Identity)); err == nil {
e.ActorKind = string(value.Kind)
}
}
d.emit(ctx, e)
}
func (d *Daemon) acceptTaskPublisher(ctx context.Context, msg eventbus.Message, name string) bool {
if ok, reason, class := d.identityMayAct(ctx, name); !ok {
d.Log.ErrorContext(ctx, "nats intake rejected: "+reason, "subject", msg.Subject())
if class == taskstate.ParkTransient {
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nak failed", "err", err)
}
return false
}
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
return false
}
return true
}
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
// and runs the workflow. The message is ack'd on terminal (park is a valid
// outcome). Nak on transient errors so NATS redelivers.
func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message, hold *taskHold) {
ctx, span := natsrpc.StartConsumer(ctx, msg.Subject(), msg.Header())
defer span.End()
tm, err := workintake.DecodeTask(msg.Data())
if err != nil {
d.Log.ErrorContext(ctx, "nats decode failed", "err", err)
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
// bad message, don't retry
return
}
if !d.acceptTaskPublisher(ctx, msg, tm.Identity) {
return
}
inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
if err != nil {
d.Log.ErrorContext(ctx, "nats enqueue failed", "err", err)
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if !inserted {
// Already tracked -- dedup. Ack and move on.
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
if err != nil {
d.Log.ErrorContext(ctx, "nats claim failed", "err", err)
if err := msg.Nak(); err != nil {
d.Log.WarnContext(ctx, "nats nak failed", "err", err)
}
return
}
if task == nil {
// Claimed by another consumer, or task is not queued. Ack.
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
return
}
d.emitTaskQueued(ctx, task)
d.process(ctx, task, hold)
if err := msg.Ack(); err != nil {
d.Log.WarnContext(ctx, "ack failed", "err", err)
}
}
// closeResolvedIssue closes a finished task's forge issue, skipping chat
// tasks. Failures are logged.
func (d *Daemon) closeResolvedIssue(ctx context.Context, fg forge.Forge, task *workflow.Task, comment string) {
if !task.IsForgeBacked() {
return
}
if d.ResolveRepository != nil {
resolved, err := d.Repository(ctx, task, config.OperationPR)
if err != nil {
d.Log.WarnContext(ctx, "issue closure not permitted", "task", task.ID)
return
}
fg = resolved.Forge
}
if err := fg.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil {
d.Log.WarnContext(ctx, "closing resolved issue failed; it stays open and will be re-polled",
"repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err)
}
}
// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state
// and cleans up their worktrees.
func (d *Daemon) reconcilePRs(ctx context.Context) error {
tasks, err := d.Store.OpenPRs(ctx)
if err != nil {
return fmt.Errorf("reconcile query: %w", err)
}
var failed []string
for _, t := range tasks {
fg := d.forgeFor(ctx, &t)
if _, unavailable := fg.(*forge.NoopForge); unavailable {
continue
}
trees := d.treesFor(&t)
// PRState/PR reconciliation is valid regardless of Source -- a
// chat-sourced task can still open a real PR against a real
// branch. Only the issue-label call (tied to the synthetic
// issue number) is forge-only.
state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber)
if err != nil {
d.Log.WarnContext(ctx, "PR state check failed", "pr", t.PRNumber, "err", err)
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
continue
}
attempt := t.Attempt
switch state {
case "merged":
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, ""); err != nil {
d.Log.ErrorContext(ctx, "PR merged but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
continue
}
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err)
}
// Close the issue explicitly; nothing else does.
d.closeResolvedIssue(ctx, fg, &t, fmt.Sprintf(
"Closed by #%d, merged by archie.", t.PRNumber,
))
d.Log.InfoContext(ctx, "PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
d.emit(ctx, events.Event{
Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt,
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
Data: map[string]any{"pr": t.PRNumber},
})
case "closed":
if err := d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge"); err != nil {
d.Log.ErrorContext(ctx, "PR rejected but task transition failed", "task", t.ID, "pr", t.PRNumber, "err", err)
failed = append(failed, fmt.Sprintf("%s/%s#%d", t.Owner, t.Repo, t.PRNumber))
continue
}
if err := trees.Cleanup(t.Owner, t.Repo, t.IssueNumber); err != nil {
d.Log.WarnContext(ctx, "worktree cleanup failed", "task", t.ID, "err", err)
}
d.Log.InfoContext(ctx, "PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
d.emit(ctx, events.Event{
Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt,
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
Data: map[string]any{"pr": t.PRNumber},
})
}
}
return passFailed("PR reconcile", failed)
}
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
// A task off the NATS stream only has an id once the store has claimed its
// row, inside this call, so its capacity is named here.
hold.name(task.ID, task.Attempt)
// The log opens before any check that can park, so every park reason
// reaches the attempt log.
defer d.openTaskLog(task)()
// The workflow may name the identity the run acts as, which everything
// below is chosen by, so it is settled before anything else.
if !d.adoptWorkflowIdentity(ctx, task) {
return
}
// Register first so the task can be stopped during setup.
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
defer finished()
if d.parkIfIdentityUnresolvable(ctx, task) {
return
}
trees, repo, err := d.taskWorkspace(ctx, task)
if err != nil {
d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err))
return
}
if d.ContainerPool == nil || d.Tasks == nil {
d.parkCapabilityUnavailable(ctx, task)
return
}
d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
workDir, ok := d.prepareWorkspace(ctx, task, trees, repo)
if !ok {
return
}
if !task.HasRepository() {
defer func() {
if err := trees.RemoveScratch(task.ID); err != nil {
d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err)
}
}()
}
// The workflow is pinned before the container starts, because the
// profile it names decides the container's image.
profile, ok := d.pinTaskProfile(ctx, task)
if !ok {
return
}
ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image)
if !ok {
return
}
defer d.ContainerPool.Release(ctx, ctr)
defer revokeCredential()
// Run the whole task in archie-agent, which calls back to the daemon for
// store, forge and push operations.
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
runCtx, stopWatch := withContainerExit(limitCtx, ctr)
d.runViaAgent(runCtx, task, repo, profile, credential)
stopWatch()
stopLimit()
d.teardownStorage(ctx, task, repo, workDir)
d.cleanupTerminalTaskWorktree(ctx, task, trees)
}
// teardownStorage runs after the workflow completes. The Docker backend is
// a no-op; future backends (temp volumes, NFS leases) use this hook.
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
if d.Storage == nil {
return
}
if err := d.Storage.Teardown(ctx, storage.TaskRef{
WorktreeDir: workDir,
Ecosystem: repo.Ecosystem,
PersistentStorage: repo.PersistentStorage,
Owner: task.Owner,
Repo: task.Repo,
}); err != nil {
d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err)
}
}
func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) {
if task.HasRepository() && d.ResolveRepository != nil {
resolved, err := d.Repository(ctx, task, config.OperationClone)
if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) {
// Checked before the run rather than at its commit step, so a
// missing permit parks in seconds instead of after the agent's work.
_, err = d.Repository(ctx, task, config.OperationPush)
}
return resolved.Trees, resolved.Repository, err
}
trees := d.treesFor(task)
repo, ok := d.repoFor(task)
if !ok && task.HasRepository() {
return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")}
}
return trees, repo, nil
}
// prepareWorkspace makes the directory the container binds as its
// workspace, which must exist before Acquire: a fresh clone of the task's
// repository, or an empty scratch directory for a task with none. It parks
// the task and reports false when it cannot.
func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) {
if !task.HasRepository() {
dir, err := trees.PrepareScratch(task.ID)
if err != nil {
d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err)
d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient)
return "", false
}
return dir, true
}
// The retry mode on the task decides whether to resume the pushed branch or
// reset onto base. Prepare clones a missing worktree either way.
target := worktree.Fresh
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryKeepWorktree {
target = worktree.Keep
// Consumed: a later plain requeue must not skip the reset.
task.RetryMode = ""
}
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork {
if task.Branch == "" {
// The action refuses this combination, so an empty branch here is a
// dispatch bug upstream and no retry can fix it.
d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID)
d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision)
return "", false
}
target = worktree.Target(task.Branch)
}
d.reclaimStaleClones(ctx, task, trees)
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
return "", false
}
// Every task gets an independent full clone.
prepare := func() (string, string, error) {
return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
}
dir, branch, err := prepare()
if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) {
dir, branch, err = prepare()
}
if err != nil {
reason := "worktree prepare failed: " + err.Error()
if target != worktree.Fresh {
reason = "worktree resume failed: " + err.Error()
}
d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
return "", false
}
task.Branch = branch
if err := d.Store.Update(ctx, task); err != nil {
d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err)
}
return dir, true
}
// reclaimStaleClones frees the work volume before a clone waits on it, by
// removing the oldest clones nothing has touched for cloneRetention. A clone
// with uncommitted or unpushed work is kept.
func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace {
return
}
n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace, func(dir string) bool { return trees.HoldsUnsharedWork(ctx, dir) })
if err != nil {
d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err)
}
if n > 0 {
d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n)
}
}
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
// holds uncommitted work or its state cannot be read.
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
if d.Store == nil || trees == nil || task == nil || !task.HasRepository() {
return
}
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
if err != nil {
d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
return
}
// A run that handed off shares its work item's worktree with its
// successor, which owns it from here.
if latest == nil || latest.HandoffToTaskID != 0 {
return
}
// Settled statuses. Keep the worktree while a PR exists.
switch latest.Status {
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead:
if latest.PRNumber != 0 {
return
}
if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) {
return
}
if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil {
d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err)
}
}
}
// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has
// to survive cleanup because it may hold work no commit captured. A worktree
// that is not on disk at all has nothing to keep, so a task whose clone was
// already removed still cleans up whatever leftovers share its path.
func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool {
dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
if _, err := os.Stat(dir); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false
}
d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
return true
}
uncommitted, err := trees.HasUncommittedChanges(ctx, dir)
if err != nil {
d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
return true
}
if uncommitted {
d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir)
return true
}
return false
}
// openTaskLog opens the task's log sink for one process() call and returns
// its closer. Failure to open is logged, not fatal.
func (d *Daemon) openTaskLog(task *workflow.Task) func() {
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
}
return func() {
if err := d.TaskLogs.Close(task.ID); err != nil {
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
}
}
}
// acquireTaskContainer prepares and acquires the sandbox container for a
// task, reporting false when it could not -- having already parked the
// task and logged why. The caller releases the container it returns.
func (d *Daemon) acquireTaskContainer(
ctx context.Context,
task *workflow.Task,
repo config.Repo,
workDir string,
image string,
) (*container.Container, string, func(), bool) {
park := func(reason string, err error) {
d.Log.ErrorContext(ctx, reason, "err", err)
d.parkBlockedTask(ctx, task, reason+": "+err.Error(), taskstate.ParkTransient, parkBlocker(err))
}
// Resolve the PR number from inputs before the brief is written.
if task.Workflow == "pr-review" {
task.PRNumber = task.EffectivePRNumber()
}
if err := writeTaskBrief(workDir, task); err != nil {
park("task.json write failed", err)
return nil, "", nil, false
}
// Fetch the pull request with the daemon's forge credential into workDir,
// so the container reads it without credentials.
if task.Workflow == "pr-review" {
if err := prefetchPRReview(ctx, d.forgeFor(ctx, task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil {
park("pr-review prefetch failed", err)
return nil, "", nil, false
}
}
// Guard: Storage may be nil if the daemon was wired incorrectly. In
// normal operation, Storage is always set when ContainerPool is set.
if d.Storage == nil {
d.Log.ErrorContext(ctx, "storage backend is nil -- cannot acquire container")
d.parkRunningTask(ctx, task, "storage backend not configured", taskstate.ParkTransient)
return nil, "", nil, false
}
mounts, err := d.Storage.Setup(ctx, storage.TaskRef{
WorktreeDir: workDir,
Ecosystem: repo.Ecosystem,
PersistentStorage: repo.PersistentStorage,
Owner: task.Owner,
Repo: task.Repo,
})
if err != nil {
park("storage setup failed", err)
return nil, "", nil, false
}
credential, revokeCredential, err := d.runCredential(ctx, task)
if err != nil {
d.teardownStorage(ctx, task, repo, workDir)
park("run credential failed", err)
return nil, "", nil, false
}
model, closeModel, err := d.openModelEgress(ctx, task, credential)
if err != nil {
revokeCredential()
d.teardownStorage(ctx, task, repo, workDir)
park("model egress failed", err)
return nil, "", nil, false
}
revokeRun := revokeCredential
revokeCredential = func() { closeModel(); revokeRun() }
// The container lives in its own network namespace, so a loopback State
// Store target names the container itself. Give this task an address on
// the host bridge that forwards to it, closed with the task.
stateStoreURL, relay, release, err := d.taskStateStoreEndpoint(ctx, revokeCredential)
if err != nil {
revokeCredential()
d.teardownStorage(ctx, task, repo, workDir)
park("state store relay failed", err)
return nil, "", nil, false
}
revokeCredential = release
ctr, err := d.ContainerPool.Acquire(ctx, image, append(mounts, model.Mounts...), append(d.containerEnv(task, credential, stateStoreURL), model.Env...))
if err != nil {
revokeCredential()
// Roll back the storage we just set up: the mounts were created
// for this task but no container will use them. Leaking them until
// a later TTL sweep is not acceptable on a backend that allocates
// real resources (volumes, NFS leases).
d.teardownStorage(ctx, task, repo, workDir)
park("container acquire failed", err)
return nil, "", nil, false
}
d.admitToRelay(ctx, relay, task, ctr)
return ctr, credential, revokeCredential, true
}
// writeTaskBrief writes task.json, the container's boot-time brief.
func writeTaskBrief(workDir string, task *workflow.Task) error {
return container.WriteTaskJSON(workDir, container.TaskPayload{
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
Labels: strings.Split(task.Labels, ","),
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
Inputs: task.Inputs,
})
}
// parkCapabilityUnavailable parks a task whose daemon-side capability is
// missing (container pool or task transport): both are environmental, so
// the park class is transient and the KindParked event carries the class
// for the dashboard. The reason distinguishes the two in logs and events.
func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task) {
var reason string
if d.ContainerPool == nil {
reason = "managed agent container pool is unavailable; refusing to run this task on the host"
d.Log.ErrorContext(ctx, "task parked: managed worker unavailable", "task", task.ID,
"hint", "enable containers and make the archie-agent image available, then retry the task")
} else {
reason = "agent task transport is unavailable; refusing to run this task on the host"
d.Log.ErrorContext(ctx, "task parked: agent task transport unavailable", "task", task.ID)
}
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
}
// parkRunningTask moves a running task to parked with class, using a
// bounded context that ignores cancellation. ErrStaleTransition is ignored;
// other errors are logged.
func (d *Daemon) parkRunningTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass) {
d.parkBlockedTask(ctx, task, reason, class, "")
}
// parkBlockedTask is parkRunningTask recording the blocker the park site
// knows, so the dashboard can link to its fix.
func (d *Daemon) parkBlockedTask(ctx context.Context, task *workflow.Task, reason string, class taskstate.ParkClass, blocker taskstate.Blocker) {
if d.Store == nil {
return
}
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err := d.Store.ParkTask(writeCtx, task.ID, workflow.StatusRunning, reason, class, blocker); err != nil {
if !errors.Is(err, storecontract.ErrStaleTransition) {
d.Log.WarnContext(ctx, "terminal park transition failed", "task", task.ID, "reason", reason, "err", err)
}
return
}
// Recorded only once the transition landed. An entry claiming the task
// parked would be a lie when another actor moved it out of running first
// -- the same race the parked event is withheld for.
d.recordPark(ctx, task.ID, reason)
d.emit(ctx, events.Event{
Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt,
Repo: strings.Trim(task.Owner+"/"+task.Repo, "/"), Issue: task.IssueNumber, Workflow: task.Workflow,
Detail: reason, Data: map[string]any{"park_class": class},
})
}
// recordPark writes a park reason to the attempt's log. Dropped when no sink
// is open.
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
d.TaskLogs.Write(ctx, taskID, logging.Entry{
Time: time.Now(),
Level: slog.LevelError.String(),
Message: "task parked: " + reason,
Fields: map[string]any{"component": "daemon", "task": taskID},
})
}
// runViaAgent hands a task to archie-agent and waits for its result. It parks
// the task only when the agent never answered or failed before recording an
// outcome.
//
// credential is the task's run credential; the agent presents it to push the
// task's branch.
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, credential string) {
cfg, err := d.taskConfig(ctx, task)
if err != nil {
d.parkRunningTask(ctx, task, err.Error(), taskstate.ParkTransient)
return
}
taskCfg := cfg.ForTask()
d.captureAttemptConfig(ctx, task, taskCfg)
var instructions workflow.Instructions
if d.TaskInstructions != nil {
instructions.Instance, instructions.Organisation, err = d.TaskInstructions.TaskInstructions(ctx, task.ID)
if err != nil {
d.parkRunningTask(ctx, task, "resolve the task's instructions: "+err.Error(), taskstate.ParkTransient)
return
}
}
req := taskrun.Request{
Instructions: instructions,
Task: task,
Repo: repo,
Cfg: taskCfg,
Providers: agentexec.ProvidersFromConfig(cfg.Providers),
MCPServers: cfg.Tools.MCPServers,
RunCredential: credential,
WorkflowDefinition: task.WorkflowDefinitionYAML,
Tools: d.toolAllowlist(ctx, task, profile.Tools),
}
req.Trace = map[string]string{}
otel.GetTextMapPropagator().Inject(ctx, propagation.MapCarrier(req.Trace))
data, err := json.Marshal(req)
if err != nil {
d.Log.ErrorContext(ctx, "taskrun encode failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient)
return
}
reply, err := d.requestTaskRun(ctx, task.ID, data)
if err != nil {
if stoppingOnly(ctx) {
d.Log.InfoContext(ctx, "daemon stopping; task left running for the next daemon to re-queue", "task", task.ID)
return
}
err = runFailure(ctx, err)
d.Log.ErrorContext(ctx, "taskrun request failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "taskrun request failed: "+err.Error(), taskstate.ParkTransient)
return
}
var resp taskrun.Response
if err := json.Unmarshal(reply, &resp); err != nil {
d.Log.ErrorContext(ctx, "taskrun decode response failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient)
return
}
if resp.Error != "" {
d.Log.ErrorContext(ctx, "taskrun run failed", "task", task.ID, "err", resp.Error)
d.parkRunningTask(ctx, task, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
return
}
// The agent applies the task's own outcome transition, so a status the
// task cannot reach from running means nothing landed: empty, unknown, or
// still running. Transient, because the cause is environmental -- a State
// Store that refused the transition -- and a retry can record one.
if !taskstate.CanTransition(taskstate.Running, resp.Status) {
reason := noOutcomeReason(resp.Status)
d.Log.ErrorContext(ctx, "taskrun run recorded no outcome", "task", task.ID, "status", resp.Status)
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
return
}
// Reached only past every failure return above, so this is always a
// fully decoded, error-free response -- a version is never recorded off
// a request that never reached an agent, or a response describing a
// parked/errored task.
if d.AgentStatus != nil && resp.AgentVersion != "" {
d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType)
}
d.Log.InfoContext(ctx, "taskrun complete", "task", task.ID, "status", resp.Status)
}
// noOutcomeReason names why a taskrun reply left the task without an outcome,
// quoting the status so the log line points at what the agent actually said.
func noOutcomeReason(status string) string {
if status == "" {
return "agent returned no status: the run could not record an outcome"
}
return fmt.Sprintf("agent returned unrecognised status %q: the run could not record an outcome", status)
}
// pinTaskProfile pins the task's workflow definition and resolves its agent
// profile, parking the task on failure.
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
if err := d.pinWorkflowDefinition(ctx, task); err != nil {
d.Log.ErrorContext(ctx, "pin workflow definition failed", "task", task.ID, "err", err)
d.parkBlockedTask(ctx, task, "pin workflow definition: "+err.Error(), pinParkClass(err), parkBlocker(err))
return config.AgentProfile{}, false
}
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
if err == nil {
// Every producer meets the declared interface here, including the
// ones that cannot read definitions when they enqueue (chat and
// dashboard spawns go through the Gateway, which never does).
err = iface.CheckInputs(task.Inputs)
}
if err == nil && iface.RepositoryMode() == workflowtask.RepositoryRequired && !task.HasRepository() {
// An optional workflow may hand off to one that requires a
// repository, because its task may carry one; a task that does not
// is parked here rather than run the target's steps in a scratch
// workspace.
err = fmt.Errorf("requires a repository and this task has none")
}
if err == nil {
var profile config.AgentProfile
if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil {
return profile, true
}
}
d.Log.ErrorContext(ctx, "agent profile unavailable", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "workflow "+task.Workflow+": "+err.Error(), taskstate.ParkNeedsDecision)
return config.AgentProfile{}, false
}
// pinWorkflowDefinition ensures the task carries a valid definition pin for
// the workflow it names and that its org has the workflow enabled. Every run
// passes through here, whatever started it, so this is the one enablement
// check for issue labels, handoffs, approvals, calls and playbooks alike.
func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error {
ctx, err := d.workflowContext(ctx, task)
if err != nil {
return err
}
if err := d.pinDefinition(ctx, task); err != nil {
return err
}
enablement, err := d.workflowEnablement(ctx)
if err != nil {
return pinFailure{taskstate.ParkTransient, fmt.Errorf("workflow enablement: %w", err)}
}
if !enablement.Enabled(task.Org, task.Workflow) {
return blockedPark{taskstate.BlockerWorkflowDisabled, pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow %s is disabled", task.Workflow)}}
}
if err := d.checkPackageTrigger(ctx, task); err != nil {
return pinFailure{taskstate.ParkNeedsDecision, err}
}
return nil
}
// pinDefinition keeps a valid pin and resolves a missing or mismatched one
// again from the active definitions.
func (d *Daemon) pinDefinition(ctx context.Context, task *workflow.Task) error {
pinned, ok, err := workflow.PinnedDefinitionID(task)
if err != nil {
return pinFailure{taskstate.ParkNeedsDecision, err}
}
if ok && pinned == task.Workflow {
return nil
}
d.runActionPlaybooks(ctx, task)
if d.WorkflowDefinitions == nil {
return d.pinWorkflowFromCollection(ctx, task, workflow.ShippedDefinitions(), 0)
}
collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx)
if err != nil {
return pinFailure{taskstate.ParkTransient, err}
}
return d.pinWorkflowFromCollection(ctx, task, collection, version)
}
// pinFailure carries the park class for a pin failure: operator-actionable
// for a bad pin or missing workflow, transient for a store error.
type pinFailure struct {
class taskstate.ParkClass
err error
}
func (e pinFailure) Error() string { return e.err.Error() }
func (e pinFailure) Unwrap() error { return e.err }
// pinParkClass reports the class a pin failure carries, defaulting to
// taskstate.ParkNeedsDecision for a cause that did not classify itself: the safe
// misread is "an operator should look at this", never "a requeue will fix it".
func pinParkClass(err error) taskstate.ParkClass {
if failure, ok := errors.AsType[pinFailure](err); ok {
return failure.class
}
return taskstate.ParkNeedsDecision
}
// blockedPark names what a park is missing alongside the failure.
type blockedPark struct {
blocker taskstate.Blocker
err error
}
func (e blockedPark) Error() string { return e.err.Error() }
func (e blockedPark) Unwrap() error { return e.err }
// parkBlocker reports the blocker err names, or empty.
func parkBlocker(err error) taskstate.Blocker {
if b, ok := errors.AsType[blockedPark](err); ok {
return b.blocker
}
if failure, ok := errors.AsType[*forge.AccessError](err); ok {
return accessBlocker(failure.Code)
}
return ""
}
// accessBlocker maps a repository access failure code onto what fixes it.
func accessBlocker(code string) taskstate.Blocker {
switch code {
case "operation_denied":
return taskstate.BlockerConnectionPermit
case "repository_not_configured", "connection_required", "connection_missing":
return taskstate.BlockerRepository
case "credential_unavailable", "account_unverified":
return taskstate.BlockerConnectionCredential
case "forge_unavailable":
return taskstate.BlockerForgeExtension
}
return ""
}
// runActionPlaybooks runs the action playbooks matching a task before it is
// pinned. Failures never fail the task; tasks already naming a workflow are
// skipped.
func (d *Daemon) runActionPlaybooks(ctx context.Context, task *workflow.Task) {
if d.Playbooks == nil || d.PlaybookLedger == nil || task.Workflow != "" {
return
}
if err := d.Playbooks.Run(ctx, d.PlaybookLedger, d.Log, playbookInput(task)); err != nil {
d.Log.WarnContext(ctx, "action playbook run failed", "task", task.ID, "err", err)
}
}
// resolveWorkflowID picks the definition to pin: a matching playbook, then
// the kind defaults. A task already naming a workflow keeps it.
func (d *Daemon) resolveWorkflowID(ctx context.Context, task *workflow.Task, available map[string]struct{}) (string, error) {
if d.Playbooks == nil || task.Workflow != "" {
return workflow.ResolveWorkflowID(task, available)
}
decision, matched := d.Playbooks.Dispatch(ctx, playbookInput(task))
if !matched {
return workflow.ResolveWorkflowID(task, available)
}
if _, defined := available[decision.Workflow]; !defined {
// Reported, not silently downgraded to the binding's choice: the
// operator bound this trigger to a named workflow, and running a
// different one under their rule is worse than parking the task.
return "", fmt.Errorf("playbook %q binds this trigger to workflow %q, which is not defined", decision.PlaybookID, decision.Workflow)
}
d.Log.InfoContext(ctx, "playbook selected workflow",
"task", task.ID, "playbook", decision.PlaybookID,
"playbook_version", decision.Version, "workflow", decision.Workflow)
return decision.Workflow, nil
}
// playbookInput builds the coordinator's dispatch input from a task row. The
// event surface is what the intake point actually knows about the originating
// issue -- the fields a `when` condition can read; it is deliberately not
// padded with values the daemon would have to invent.
func playbookInput(task *workflow.Task) playbook.DispatchInput {
labels := workintake.SplitLabels(task.Labels)
kind := string(workintake.KindForLabels(labels))
return playbook.DispatchInput{
Labels: labels,
Kind: kind,
TaskID: workintake.TaskEnvelope{Org: task.Org, Identity: task.Identity, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber}.IdempotencyKey(),
Event: map[string]any{
"kind": kind,
"labels": labels,
"owner": task.Owner,
"repo": task.Repo,
"number": task.IssueNumber,
"title": task.Title,
"body": task.Body,
"source": task.Source,
},
}
}
func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.Task, collection workflow.WorkflowDefinitionCollection, version int64) error {
available := make(map[string]struct{}, len(collection.Definitions))
for _, definition := range collection.Definitions {
available[definition.ID] = struct{}{}
}
id, err := d.resolveWorkflowID(ctx, task, available)
if err != nil {
return pinFailure{taskstate.ParkNeedsDecision, err}
}
definition, ok := collection.DefinitionByID(id)
if !ok {
return pinFailure{taskstate.ParkNeedsDecision, fmt.Errorf("workflow definition %q disappeared", id)}
}
task.Workflow = id
task.WorkflowDefinitionVersion = version
task.WorkflowDefinitionYAML = definition.YAML
task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML)
if err := d.Store.Update(ctx, task); err != nil {
return pinFailure{taskstate.ParkTransient, fmt.Errorf("persist workflow definition pin: %w", err)}
}
return nil
}
const (
// defaultTaskRunReadyTimeout is how long requestTaskRun retries a
// nats.ErrNoResponders before giving up, when Daemon.TaskRunReadyTimeout
// is unset.
defaultTaskRunReadyTimeout = 20 * time.Second
// defaultTaskRunRetryBackoff is the delay between retry attempts, when
// Daemon.TaskRunRetryBackoff is unset.
defaultTaskRunRetryBackoff = 250 * time.Millisecond
)
func (d *Daemon) taskRunReadyTimeout() time.Duration {
if d.TaskRunReadyTimeout > 0 {
return d.TaskRunReadyTimeout
}
return defaultTaskRunReadyTimeout
}
func (d *Daemon) taskRunRetryBackoff() time.Duration {
if d.TaskRunRetryBackoff > 0 {
return d.TaskRunRetryBackoff
}
return defaultTaskRunRetryBackoff
}
// requestTaskRun publishes the taskrun request, retrying on
// nats.ErrNoResponders until the container subscribes. Other errors return
// at once.
func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) {
subject := agentnats.SubjectForTask(taskID)
deadline := time.Now().Add(d.taskRunReadyTimeout())
backoff := d.taskRunRetryBackoff()
for {
reply, err := d.Tasks.Request(ctx, subject, data)
if err == nil {
return reply, nil
}
if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) {
return nil, err
}
wait := min(backoff, time.Until(deadline))
if wait <= 0 {
return nil, err
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(wait):
}
}
}
// taskStateStoreEndpoint is the State Store address one task container is
// handed, the relay carrying it (nil when none is needed) and the release that
// ends that relay. The container lives
// in its own network namespace, so a loopback target names the container
// itself; behind a relay, the address is the host bridge's.
func (d *Daemon) taskStateStoreEndpoint(ctx context.Context, release func()) (string, *stateStoreRelay, func(), error) {
relay, err := d.openTaskStateStoreRelay(ctx)
if err != nil {
return "", nil, release, err
}
if relay == nil {
return d.ConnectedStateStore.URL, nil, release, nil
}
return relay.Addr(), relay, func() { relay.Close(); release() }, nil
}
// admitToRelay lets ctr, and only ctr, through relay. A container whose
// addresses cannot be read is refused rather than let anyone through.
func (d *Daemon) admitToRelay(ctx context.Context, relay *stateStoreRelay, task *workflow.Task, ctr *container.Container) {
if relay == nil {
return
}
addrs, err := d.ContainerPool.ContainerAddrs(ctx, ctr.ID)
if err != nil {
d.Log.WarnContext(ctx, "state store relay: container addresses unknown, refusing its connections", "task", task.ID, "err", err)
}
relay.Admit(addrs)
}
// openTaskStateStoreRelay starts the relay that carries one task container's
// State Store traffic to the daemon's target. The target stays bound where the
// operator put it; the container is handed an address on the host bridge, the
// same way it is handed NATS's. Nil when the target already reaches the
// containers, or when no bridge address is known.
func (d *Daemon) openTaskStateStoreRelay(ctx context.Context) (*stateStoreRelay, error) {
addr, loopback := loopbackStateStoreTarget(d.ConnectedStateStore.URL)
if !loopback {
return nil, nil
}
gateway := ""
if d.ContainerPool != nil {
gateway = d.ContainerPool.HostGateway()
}
if gateway == "" {
d.Log.WarnContext(ctx, "state store target is loopback and the container pool has no host bridge address; task containers cannot reach it",
"target", d.ConnectedStateStore.URL)
return nil, nil
}
return openStateStoreRelay(ctx, gateway, addr)
}
// containerEnv is the environment a task container starts with. stateStoreURL
// is the address the container can reach the State Store by, which is the
// relay's rather than the daemon's own target.
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken, stateStoreURL string) []string {
var env []string
// The endpoint the daemon's own client connected with at startup, not
// the live config: a reloaded [nats] section must not point new
// containers at a server the daemon is not publishing on.
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
if token := d.ConnectedNATS.ContainerToken(); token != "" {
env = append(env, "NATS_TOKEN="+token)
}
// Pass the container-reachable endpoint with this task's scoped token,
// never the daemon's own.
if stateStoreURL != "" {
env = append(env, "STATE_STORE_URL="+stateStoreURL)
if len(d.ConnectedStateStore.CA) > 0 {
env = append(env, "STATE_STORE_CA="+string(d.ConnectedStateStore.CA))
}
if stateStoreToken != "" {
env = append(env, "STATE_STORE_TOKEN="+stateStoreToken)
}
}
// The agent runs as root in the container, so its commits land owned by
// UID 0 on the host. The daemon's UID/GID let the agent hand the worktree
// back before the daemon reads it to push.
env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()))
env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid()))
return env
}
// runCredential issues the task's run credential, or "" when no State Store
// is configured. It fails when no issuer is wired.
func (d *Daemon) runCredential(ctx context.Context, task *workflow.Task) (string, func(), error) {
if d.ConnectedStateStore.URL == "" {
return "", func() {}, nil
}
if d.RunCredentials == nil {
return "", nil, fmt.Errorf("state store is configured but no run credential issuer is wired")
}
return d.RunCredentials.Issue(ctx, task, d.configFor(task).Budgets.TaskWallClock.Std())
}
func (d *Daemon) configFor(task *workflow.Task) config.Config {
cfg := d.Cfg.Get()
if !d.rootTask(task) {
cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{}
cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil
cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil
}
return cfg
}
// captureAttemptConfig records the attempt's effective configuration as an
// event. Failures are logged.
func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) {
if d.Store == nil {
return
}
document, err := json.Marshal(cfg)
if err != nil {
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
return
}
event := events.Event{
At: time.Now().UTC(),
Kind: events.KindConfigCaptured,
TaskID: task.ID,
Repo: task.Owner + "/" + task.Repo,
Issue: task.IssueNumber,
Workflow: task.Workflow,
Attempt: task.Attempt,
Data: map[string]any{
"schema": events.ConfigCapturedSchema,
// Raw so the document is embedded once, as it was encoded, rather
// than as a string a reader would have to decode again.
"document": json.RawMessage(document),
},
}
id, err := d.Store.InsertEvent(ctx, event)
if err != nil {
d.Log.WarnContext(ctx, "attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
return
}
// The assigned ID tells the event sink to broadcast without inserting a
// duplicate row.
event.ID = id
d.emit(ctx, event)
}
// identityMayAct reports whether a task's identity may act, and the park
// class if not. An empty, unknown or inactive identity never falls back to
// the System forge.
func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) {
if name == "" {
return false, "task has no identity", taskstate.ParkNeedsDecision
}
id := identity.IdentityID(name)
if d.IdentityRepository != nil {
value, err := d.IdentityRepository.Get(ctx, id)
if err != nil {
return false, "execution identity unavailable: " + err.Error(), taskstate.ParkTransient
}
if value.Lifecycle != identity.LifecycleActive {
return false, "identity may not act", taskstate.ParkNeedsDecision
}
return true, "", ""
}
if name == string(identity.SystemID) {
return true, "", ""
}
return false, "identity " + name + " no longer resolves", taskstate.ParkTransient
}
// parkIfIdentityUnresolvable parks task when its identity cannot be acted on,
// reporting whether it did.
func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool {
ok, reason, class := d.identityMayAct(ctx, task.Identity)
if ok {
return false
}
d.parkRunningTask(ctx, task, reason, class)
return true
}
func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool {
if d.IdentityRepository == nil {
return true
}
if id == "" {
d.Log.ErrorContext(ctx, "identity lifecycle check failed", "err", "identity ID is empty")
return false
}
value, err := d.IdentityRepository.Get(ctx, id)
if err != nil {
d.Log.ErrorContext(ctx, "identity lifecycle check failed", "identity", id, "err", err)
return false
}
return value.Lifecycle == identity.LifecycleActive
}
// forgeFor returns the forge client that owns task: its organisation
// connection, else the root d.Forge for the root alone. This is the safety
// boundary that keeps one identity's forge token from being used against
// another identity's repos.
func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge {
if d.ResolveRepository != nil && task.HasRepository() {
resolved, err := d.Repository(ctx, task, config.OperationClone)
if err != nil {
return forge.NewNoop(d.Log)
}
return resolved.Forge
}
if !d.rootTask(task) {
return forge.NewNoop(d.Log)
}
return d.Forge
}
// treesFor returns the worktree manager that owns task, mirroring forgeFor.
func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager {
if d.ResolveRepository != nil && task.HasRepository() {
// Cleanup needs only the org's path, not a still-valid external credential.
return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())}
}
if !d.rootTask(task) {
return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))}
}
return d.Trees
}
func (d *Daemon) rootTask(task *workflow.Task) bool {
return identity.IdentityID(task.Identity) == identity.SystemID
}
// repoFor resolves a root task's repo config from Cfg.Repos.
func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) {
if !d.rootTask(t) {
return config.Repo{}, false
}
for _, r := range d.Cfg.Get().Repos {
if r.Owner == t.Owner && r.Name == t.Repo {
return r, true
}
}
return config.Repo{}, false
}
// allowConcurrentForTask reports whether the task's repo allows concurrent
// tasks. Unknown repos do not.
func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
if !task.HasRepository() {
return true
}
repo, ok := d.repoFor(task)
return ok && repo.AllowConcurrent
}
// StalestLoop reports the run loop that least recently began a pass and when,
// or the zero time when no loop has started yet. The systemd watchdog beats
// only while this is fresh, so every loop Run starts must mark its passes.
func (d *Daemon) StalestLoop() (string, time.Time) {
var loop string
var oldest int64
d.loopPasses.Range(func(k, v any) bool {
name, okName := k.(string)
if ns, ok := v.(int64); ok && okName && (oldest == 0 || ns < oldest) {
loop, oldest = name, ns
}
return true
})
if oldest == 0 {
return "", time.Time{}
}
return loop, time.Unix(0, oldest).UTC()
}
// markPass stamps the start of one loop's pass, so a hung pass shows as stale.
func (d *Daemon) markPass(loop string) {
d.loopPasses.Store(loop, time.Now().UnixNano())
}
// errContainerExited is the cancellation cause when a task's agent container
// stops before answering its taskrun request.
var errContainerExited = errors.New("agent container exited before the task finished")
// runFailure names why a taskrun request failed: the container's exit or the
// time limit when either cancelled it, else the request's own error.
func runFailure(ctx context.Context, err error) error {
if cause := context.Cause(ctx); errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit) {
return cause
}
return err
}
// exitWatcher is a running agent container, seen only for how it ends.
type exitWatcher interface {
Exited() <-chan struct{}
ExitDetail() string
}
// withContainerExit derives a context cancelled when exited closes. A dead
// container never answers its taskrun request, so without this the request
// blocks forever and the task row stays running.
func withContainerExit(ctx context.Context, ctr exitWatcher) (context.Context, func()) {
runCtx, cancel := context.WithCancelCause(ctx)
go func() {
select {
case <-ctr.Exited():
if detail := ctr.ExitDetail(); detail != "" {
cancel(fmt.Errorf("%w: %s", errContainerExited, detail))
return
}
cancel(errContainerExited)
case <-runCtx.Done():
}
}()
return runCtx, func() { cancel(nil) }
}
// errTaskTimeLimit is the cancellation cause when a task outruns its overall
// time limit (the execution settings' max task runtime).
var errTaskTimeLimit = errors.New("task exceeded its time limit")
// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies
// no bound.
func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) {
if limit <= 0 {
return ctx, func() {}
}
return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit))
}
// RemoveWorktree deletes a task's clone from the trees of the identity that
// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold
// uncaptured work, and the operator's archive is the decision to discard it.
func (d *Daemon) RemoveWorktree(task *workflow.Task) error {
trees := d.treesFor(task)
if trees == nil {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber)
}
// cleanupWorktree removes a task's clone, first handing back any root-owned
// files a killed container left in it.
func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error {
err := trees.Cleanup(owner, repo, issue)
if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) {
return trees.Cleanup(owner, repo, issue)
}
return err
}
// reownDenied repairs dir when err says the daemon was refused access to it,
// and reports whether the caller should retry. A SIGKILLed task container
// leaves root-owned files that only a container can chown back.
func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool {
if err == nil || !errors.Is(err, fs.ErrPermission) || d.ContainerPool == nil {
return false
}
if rerr := d.ContainerPool.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil {
d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr)
return false
}
return true
}
// InFlight counts the same submitted work that graceful shutdown waits for,
// including tasks waiting for a repository or concurrency slot.
func (d *Daemon) InFlight() int {
dispatcher := d.taskDispatcher()
dispatcher.mu.Lock()
defer dispatcher.mu.Unlock()
return dispatcher.pending
}