prepareWorkspace chose between Manager.Prepare (refresh onto base) and a separate Manager.Resume, and Manager.Resume only opened an already-prepared clone. A remediation whose local worktree the terminal cleanup removed, an expired volume dropped, or a different host never had could therefore never be resumed, and the retry path had no way to select the mode that continues pushed work. Unify the two matched methods into Prepare(..., Target): Fresh resets onto origin/<base>, a non-empty Target resets onto origin/<that branch>, and either target clones a missing worktree. The daemon's retry path selects the target from the task row and passes the persisted branch, so a resume lands on the branch the open PR lives on and fails closed when it is not on the remote. Delete the container-mode no-op resume (hybridTrees.Resume and the resumableTrees capability) now that the daemon owns it. The remediate workflow no longer stages a resume, and the retired remediate.resume step word stays resolvable as an inert stage: a stored definition is validated as a whole collection, so a removed word would fail every workflow to decode, not just its own.
2487 lines
98 KiB
Go
2487 lines
98 KiB
Go
// Package daemon is archied's resident loop: poll GitHub for labelled
|
|
// issues, enqueue them, and process tasks through their routed workflows.
|
|
// State lives in the store; the daemon is restartable at any point.
|
|
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"log/slog"
|
|
"maps"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/container"
|
|
"github.com/samcharles93/archie-core/internal/domain/access"
|
|
"github.com/samcharles93/archie-core/internal/domain/binding"
|
|
"github.com/samcharles93/archie-core/internal/domain/curator"
|
|
"github.com/samcharles93/archie-core/internal/domain/eda/playbook"
|
|
"github.com/samcharles93/archie-core/internal/domain/identity"
|
|
"github.com/samcharles93/archie-core/internal/domain/mapping"
|
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
|
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
|
"github.com/samcharles93/archie-core/internal/eventbus"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/forge"
|
|
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
|
"github.com/samcharles93/archie-core/internal/logging"
|
|
"github.com/samcharles93/archie-core/internal/plugin"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
"github.com/samcharles93/archie-core/internal/taskrun"
|
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
|
"github.com/samcharles93/archie-core/internal/tools"
|
|
"github.com/samcharles93/archie-core/internal/worktree"
|
|
)
|
|
|
|
// TaskBus is the messaging the daemon needs: announce discovered work,
|
|
// collect it back, and ask a worker to run a task.
|
|
//
|
|
// Declared here rather than taking eventbus.Bus because a domain defines the
|
|
// smallest interface required to do its work -- the daemon never subscribes
|
|
// or creates reply inboxes.
|
|
type TaskBus interface {
|
|
PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error
|
|
Fetch(ctx context.Context) (eventbus.Message, error)
|
|
Request(ctx context.Context, subject string, payload []byte) ([]byte, error)
|
|
}
|
|
|
|
// WorktreeGrantIssuer gives one container dispatch the narrow authority to
|
|
// publish its already-prepared branch. The daemon owns grant lifetime; the
|
|
// transport implementation owns token mechanics.
|
|
type WorktreeGrantIssuer interface {
|
|
Issue(task *workflow.Task) (token string, revoke func(), err error)
|
|
}
|
|
|
|
// StateStoreGrantIssuer gives one container dispatch a State Store credential
|
|
// scoped to that task's own workflow.Store RPCs (Update, Transition,
|
|
// InsertEvent) -- never the daemon's own administrative credential. The
|
|
// daemon owns grant lifetime (issue before Acquire, revoke on Release); the
|
|
// transport implementation (staterpc.GrantIssuer) owns token mechanics.
|
|
type StateStoreGrantIssuer interface {
|
|
Issue(task *workflow.Task) (token string, revoke func(), err error)
|
|
}
|
|
|
|
// NATSEndpoint is the broker address and credential archied connected with at
|
|
// startup. It is runtime state rather than configuration: embedded mode
|
|
// generates both values, and external secret references are resolved once.
|
|
type NATSEndpoint struct {
|
|
URL string
|
|
Token string
|
|
}
|
|
|
|
// StateStoreEndpoint is the State Store gRPC target (and the bearer token the
|
|
// daemon presents, when one is configured) that containerEnv injects as
|
|
// STATE_STORE_URL / STATE_STORE_TOKEN so archie-agent authenticates to the
|
|
// same remote archie-state-store service the daemon dials
|
|
// (docs/prds/state-store-contract.md §6, §9, §10). After the in-process
|
|
// serving path is deleted (docs/prds/state-store-contract.md §12 step 7) this
|
|
// is always the configured [services.state].target, never a daemon-owned
|
|
// listener.
|
|
type StateStoreEndpoint struct {
|
|
URL string
|
|
Token string
|
|
}
|
|
|
|
type Daemon struct {
|
|
// Cfg publishes the running configuration. Read through Cfg() so a
|
|
// reload swaps the published snapshot atomically. See config.Holder.
|
|
Cfg *config.Holder
|
|
// ConnectedNATS is the endpoint and credential the daemon's own client
|
|
// connected with at startup. Container env is built from this, not from
|
|
// live config or environment, so reload cannot point new containers at a
|
|
// broker the daemon is not publishing on. The zero value is invalid in
|
|
// production composition and remains useful only to fail closed in tests.
|
|
ConnectedNATS NATSEndpoint
|
|
// ConnectedStateStore is the State Store gRPC target (and token) the
|
|
// daemon's own client dialed at startup. containerEnv injects these as
|
|
// STATE_STORE_URL / STATE_STORE_TOKEN so archie-agent reaches the same
|
|
// remote archie-state-store service; it is always the configured
|
|
// [services.state].target after the in-process serving path is deleted.
|
|
ConnectedStateStore StateStoreEndpoint
|
|
Store storecontract.TaskStore
|
|
// Mappings persists payload field mappings (docs/prds/payload-field-mapping.md).
|
|
// Used by the binding dispatch loop to resolve capture bodies against
|
|
// the mapping a binding names. Optional: nil disables the binding
|
|
// dispatch loop (legacy behaviour).
|
|
Mappings storecontract.MappingStore
|
|
// Bindings persists playbook bindings (docs/prds/playbook-binding.md).
|
|
// Optional: nil disables the binding dispatch loop (legacy behaviour).
|
|
Bindings storecontract.BindingStore
|
|
// BindingDispatcher is the dispatch-time helper surface for bindings:
|
|
// list undispatched captures, look up armed bindings by source for the
|
|
// matcher, and write the at-most-once dedup ledger row. Split from
|
|
// BindingStore to keep the CRUD interface under the interfacebloat
|
|
// limit. Optional: nil disables the dispatch loop.
|
|
BindingDispatcher storecontract.BindingDispatcher
|
|
// MappingMatches counts each event a mapping resolved at dispatch.
|
|
// Optional: nil leaves match counts unrecorded.
|
|
MappingMatches storecontract.MappingMatchRecorder
|
|
// BindingTaskCreator enqueues a task triggered by a binding. Split
|
|
// off TaskLifecycle so the lifecycle surface stays narrow and the
|
|
// dispatch loop does not acquire the full task-creation contract.
|
|
// Optional: nil disables the dispatch loop.
|
|
BindingTaskCreator storecontract.BindingTaskCreator
|
|
// Access evaluates the policy chain at dispatch, the second of the two
|
|
// Authorizer call sites (docs/prds/orgs-and-access.md, "Where it
|
|
// lives"): the workflow's identity may `run` the workflow in its
|
|
// workspace. Optional: nil dispatches without the chain, which is the
|
|
// behaviour of an install that has not built it.
|
|
Access access.Authorizer
|
|
// Principals assembles the dispatch principal for the workflow's
|
|
// identity. Wired with Access or not at all.
|
|
Principals access.PrincipalSource
|
|
// Denials records dispatch refusals. Optional: nil skips the record.
|
|
Denials access.DenialStore
|
|
Forge forge.Forge
|
|
Trees *worktree.Manager
|
|
Bus *events.Bus
|
|
Log *slog.Logger
|
|
// Tasks is the NATS task distribution bus: the poller publishes
|
|
// discovered work over it and runViaAgent requests execution through it.
|
|
// NATS startup is mandatory (there is no broker-off execution mode), so
|
|
// production composition always sets it. Nil appears only in tests and
|
|
// is handled fail-closed at the publish/run sites.
|
|
Tasks TaskBus
|
|
// reactionConsumer is built lazily by drainReactions and kept for its
|
|
// lifetime drop counter.
|
|
reactionConsumer *reactionConsumer
|
|
// reactionPublisher is built lazily by scanPRReviews; an indirection
|
|
// only so tests can swap the delivery without the bus.
|
|
reactionPublisher ReactionPublisher
|
|
// Reactions pulls the review-reaction fan-out stream
|
|
// (docs/prds/pr-review-remediation.md). Each cycle it is drained into
|
|
// queued remediate runs before the task drain, so a reaction can be
|
|
// claimed in the same pass that carried it. Optional: nil disables the
|
|
// reaction consumer (tests, or a deployment without the reaction
|
|
// stream).
|
|
Reactions ReactionSource
|
|
WorktreeGrants WorktreeGrantIssuer
|
|
// StateStoreGrants issues per-task State Store credentials for container
|
|
// env (see containerEnv). Required whenever ConnectedStateStore.URL is
|
|
// set -- acquireTaskContainer parks the task rather than fall back to
|
|
// forwarding the daemon's own administrative token.
|
|
StateStoreGrants StateStoreGrantIssuer
|
|
// KitLauncher starts tasks whose agent profile is a Kit. Nil parks them.
|
|
KitLauncher KitLauncher
|
|
// TaskRunReadyTimeout bounds how long runViaAgent retries an initial
|
|
// taskrun request that fails with nats.ErrNoResponders, giving a
|
|
// freshly spawned archie-agent container time to connect to NATS, set
|
|
// up its task-scoped core-NATS subscription, and become reachable before the daemon
|
|
// gives up and parks the task. ContainerPool.Acquire returns as soon
|
|
// as Docker has issued the start syscall, not once that setup
|
|
// finishes, so without this bound the very first request after a
|
|
// container spawn fails deterministically. Zero uses
|
|
// defaultTaskRunReadyTimeout.
|
|
TaskRunReadyTimeout time.Duration
|
|
// TaskRunRetryBackoff is the delay between retry attempts within
|
|
// TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff.
|
|
TaskRunRetryBackoff time.Duration
|
|
// AgentStatus records the version/install-type the most recently
|
|
// completed archie-agent task response reported about itself, for
|
|
// releaseupdate.Report.Verify to check an update claim against. Nil
|
|
// disables observation -- runViaAgent skips the Observe call rather
|
|
// than reporting the agent as unverifiable via a zero value, which
|
|
// would read as a checked "unknown" rather than "never wired".
|
|
AgentStatus *AgentStatus
|
|
|
|
// ContainerPool manages Docker container lifecycle. Nil means autonomous
|
|
// execution is unavailable and process parks tasks; every runnable task gets
|
|
// a fresh container.
|
|
ContainerPool *container.Pool
|
|
// Storage is the pluggable storage backend for container mounts.
|
|
// A runnable task requires it; acquireTaskContainer parks on nil.
|
|
Storage storage.Backend
|
|
// CapabilityHost owns validated plugin manifests and cross-family
|
|
// lifecycle. Typed capability registries remain in their domain packages;
|
|
// the host never exposes daemon internals or an untyped service locator.
|
|
CapabilityHost *plugin.Host
|
|
// Curators is the curator engine family registry (epic archie-core-yp9).
|
|
// The runtime loop (archie-core-89x) and reference curators (i7i, gs8)
|
|
// are driven through it; nil when the family is not composed.
|
|
Curators *curator.Registry
|
|
// Identities holds per-identity runner state for multi-identity mode.
|
|
// When non-empty, Run() starts one goroutine per identity instead of
|
|
// using the single-identity Forge/Trees/Cfg.Repos path.
|
|
Identities []*IdentityRunner
|
|
// IdentityRepository is the durable lifecycle authority. Polling checks it
|
|
// every cycle so suspend/reactivate commands do not require a restart.
|
|
IdentityRepository identity.Repository
|
|
RootIdentityID identity.IdentityID
|
|
|
|
// Guardrails is the tool-call guardrail engine, wired by the composition
|
|
// root. When non-nil, tool successes and failures are recorded and
|
|
// warnings/hard-stops are issued per the configured thresholds. Nil
|
|
// means guardrails are disabled (backward compatible).
|
|
Guardrails *tools.GuardrailEngine
|
|
|
|
// KindWorkflows and LabelWorkflows are the resolved kind/label ->
|
|
// workflow-name routing bindings loaded at startup (WorkflowRoutingFile,
|
|
// WorkflowLabelsFile, PlaybookDirs). They travel in taskrun.Request so
|
|
// the archie-agent process that calls workflow.Route applies the same
|
|
// bindings the daemon loaded; nil means built-in defaults.
|
|
KindWorkflows workflow.KindWorkflows
|
|
LabelWorkflows workflow.LabelWorkflows
|
|
// Playbooks is the loaded EDA playbook set (workflow and action
|
|
// playbooks, internal/domain/eda/playbook). It is consulted before the
|
|
// kind/label bindings when a task's workflow definition is pinned: a
|
|
// workflow playbook is an operator's explicit rule for one trigger, where
|
|
// the bindings are a table of defaults. Nil means no playbooks are loaded
|
|
// and routing is exactly the binding behaviour.
|
|
Playbooks interface {
|
|
Dispatch(playbook.DispatchInput) (playbook.Decision, bool)
|
|
Run(context.Context, storecontract.PlaybookDispatcher, *slog.Logger, playbook.DispatchInput) error
|
|
}
|
|
// PlaybookLedger is the at-most-once gate action playbooks run through.
|
|
// Nil means action playbooks do not run: they never fire unguarded.
|
|
PlaybookLedger storecontract.PlaybookDispatcher
|
|
|
|
// WorkflowDefinitions supplies the active database definitions. A task is
|
|
// pinned once before dispatch; retries reuse the task's stored YAML.
|
|
WorkflowDefinitions interface {
|
|
WorkflowDefinitions(context.Context) (workflow.WorkflowDefinitionCollection, int64, error)
|
|
}
|
|
// WorkflowEnablement supplies which workflows each org has disabled. A
|
|
// binding targeting a workflow its org disabled does not dispatch. Nil
|
|
// means every workflow is enabled.
|
|
WorkflowEnablement interface {
|
|
WorkflowEnablement(context.Context) (workflowtask.WorkflowEnablement, error)
|
|
}
|
|
|
|
// ToolRegistry is the central tool registry, wired by the composition
|
|
// root. MCP-discovered tools and built-in tools are registered here
|
|
// and passed as CaptureTools in agent requests. Nil means no dynamic
|
|
// tool discovery (backward compatible).
|
|
ToolRegistry *tools.Registry
|
|
|
|
// TaskLogs persists each task's own log output (including a sandboxed
|
|
// container's, which otherwise disappears at AutoRemove) and mirrors it
|
|
// live onto the dashboard feed, wired by the composition root. Nil
|
|
// disables task logging (backward compatible); every TaskRegistry
|
|
// method is nil-receiver-safe, so process() calls it unconditionally.
|
|
TaskLogs *logging.TaskRegistry
|
|
|
|
// running holds a cancel function for every task currently executing,
|
|
// so /stop can reach work already in flight. Its zero value is ready
|
|
// to use.
|
|
running runningTasks
|
|
|
|
// lastPollAt is when the most recent poll pass began, in Unix
|
|
// nanoseconds (zero = no pass has started yet), read by
|
|
// LastPollAt. Atomic because runIdentities runs one poll goroutine per
|
|
// identity and they all stamp this one "is the poller alive" reading.
|
|
lastPollAt atomic.Int64
|
|
}
|
|
|
|
// IdentityRunner bundles identity-specific state for a single agent
|
|
// identity running within a multi-identity daemon. Each identity gets its
|
|
// own forge client, worktree manager, repo list, and config -- but shares
|
|
// the store, NATS connection, container pool, and event bus with siblings.
|
|
type IdentityRunner struct {
|
|
ID identity.IdentityID
|
|
Name string
|
|
Forge forge.Forge
|
|
Trees *worktree.Manager
|
|
Repos []config.Repo
|
|
// Cfg is the identity-scoped config subset (forge, models, dispatch,
|
|
// budgets, etc.).
|
|
Cfg config.IdentityConfig
|
|
Log *slog.Logger
|
|
}
|
|
|
|
// NewIdentityRunner constructs an IdentityRunner from an identity config
|
|
// and a pre-built forge client. The caller owns forge and trees lifecycle;
|
|
// IdentityRunner just holds references.
|
|
func NewIdentityRunner(ctx context.Context, idCfg config.IdentityConfig, fg forge.Forge, trees *worktree.Manager, log *slog.Logger) (*IdentityRunner, error) {
|
|
if idCfg.Name == "" {
|
|
return nil, fmt.Errorf("identity name is required")
|
|
}
|
|
return &IdentityRunner{
|
|
ID: identity.StableID(idCfg.Name),
|
|
Name: idCfg.Name,
|
|
Forge: fg,
|
|
Trees: trees,
|
|
Repos: idCfg.Repos,
|
|
Cfg: idCfg,
|
|
Log: log.With("identity", idCfg.Name),
|
|
}, nil
|
|
}
|
|
|
|
// emit publishes an observability event; safe on a nil bus.
|
|
func (d *Daemon) emit(e events.Event) {
|
|
if d.Bus != nil {
|
|
d.Bus.Publish(e)
|
|
}
|
|
}
|
|
|
|
// Startup runs crash recovery and access verification once.
|
|
func (d *Daemon) Startup(ctx context.Context) error {
|
|
d.cleanupExpiredStorage(ctx)
|
|
n, err := d.Store.RecoverStale(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
d.Log.Info("re-queued tasks left running by a previous daemon", "count", n)
|
|
}
|
|
|
|
d.sweepAccess(ctx)
|
|
return nil
|
|
}
|
|
|
|
// sweepTimeout bounds one forge's boot sweep. Startup gates the gateways and
|
|
// the run loop, and a forge client carries no request deadline of its own, so
|
|
// an unreachable host would otherwise hold boot open indefinitely.
|
|
const sweepTimeout = 2 * time.Minute
|
|
|
|
// sweepAccess accepts pending invitations and verifies push access for the
|
|
// root forge and for every configured identity's forge.
|
|
//
|
|
// The root is swept even in multi-identity mode. Run takes runIdentities
|
|
// there and never polls the root repo list, but polling is not the only way a
|
|
// task reaches a repo: dispatchBinding enqueues with an empty identity and
|
|
// resolveBindingRepo falls back to the root repo list, and forgeFor/repoFor
|
|
// resolve an empty identity to the root forge and root repos. Root targets
|
|
// stay reachable, so they stay worth verifying; a deployment with no root
|
|
// forge configured gets the noop client, whose sweep is silent.
|
|
//
|
|
// Each forge is swept in its own goroutine under its own deadline, mirroring
|
|
// Run's goroutine-per-identity isolation. Warnings alone isolate a failing
|
|
// sibling; they do not isolate a stalled one.
|
|
func (d *Daemon) sweepAccess(ctx context.Context) {
|
|
sweep := func(identity string, fg forge.Forge, repos []config.Repo) {
|
|
log := d.Log
|
|
if identity != "" {
|
|
log = log.With("identity", identity)
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, sweepTimeout)
|
|
defer cancel()
|
|
if err := fg.AcceptInvitations(ctx); err != nil {
|
|
log.Warn("invitation sweep failed", "err", err)
|
|
}
|
|
for _, r := range repos {
|
|
if err := fg.VerifyPush(ctx, r.Owner, r.Name); err != nil {
|
|
// A sweep cut short by its own context is not a repo failure: name
|
|
// the budget when it expired, stay quiet on a shutdown.
|
|
switch {
|
|
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
|
// Every later call would return the same error, so say it once.
|
|
log.Warn("push check abandoned: sweep budget exhausted", "repo", r.FullName(), "err", err)
|
|
return
|
|
case ctx.Err() != nil:
|
|
return
|
|
}
|
|
log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(1 + len(d.Identities))
|
|
go func() {
|
|
defer wg.Done()
|
|
sweep("", d.Forge, d.Cfg.Get().Repos)
|
|
}()
|
|
for _, id := range d.Identities {
|
|
go func() {
|
|
defer wg.Done()
|
|
sweep(id.Name, id.Forge, id.Repos)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
// Run polls until the context ends. When Identities is non-empty, each
|
|
// identity gets its own goroutine with its own forge client, repo list,
|
|
// and poll interval -- failure isolation between identities (one identity's
|
|
// forge outage doesn't block another's poll cycle).
|
|
func (d *Daemon) Run(ctx context.Context) error {
|
|
if len(d.Identities) > 0 {
|
|
return d.runIdentities(ctx)
|
|
}
|
|
interval := d.Cfg.Get().PollInterval.Std()
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
d.Cycle(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
// PollInterval may have been reloaded; reset the ticker so a
|
|
// change takes effect on the next cycle.
|
|
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
|
|
interval = iv
|
|
ticker.Reset(iv)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// runIdentities starts one goroutine per identity plus one shared
|
|
// maintenance-and-drain loop.
|
|
//
|
|
// Polling is identity-scoped: each identity goroutine polls only its own
|
|
// repos via its own forge client and enqueues under its own name. It never
|
|
// drains or reconciles. Maintenance and draining are store-wide, not
|
|
// identity-scoped, so they run once in a single shared loop -- that keeps
|
|
// the global max_concurrency cap and the per-repo serialization intact
|
|
// across identities (N per-identity dispatchers would multiply the cap and
|
|
// split the same-repo lock), and it ensures reconcilePRs and storage
|
|
// cleanup actually run in multi-identity mode.
|
|
func (d *Daemon) runIdentities(ctx context.Context) error {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
// One poll loop per identity. Polling is the only identity-scoped
|
|
// work; it never executes tasks.
|
|
for _, id := range d.Identities {
|
|
wg.Add(1)
|
|
go func(id *IdentityRunner) {
|
|
defer wg.Done()
|
|
interval := d.Cfg.Get().PollInterval.Std()
|
|
if id.Cfg.PollInterval > 0 {
|
|
interval = id.Cfg.PollInterval.Std()
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
d.pollForIdentity(ctx, id)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
// The identity's own override is frozen (identity config is
|
|
// startup-built); the root fallback may have been reloaded.
|
|
iv := d.Cfg.Get().PollInterval.Std()
|
|
if id.Cfg.PollInterval > 0 {
|
|
iv = id.Cfg.PollInterval.Std()
|
|
}
|
|
if iv != interval {
|
|
interval = iv
|
|
ticker.Reset(iv)
|
|
}
|
|
}
|
|
}
|
|
}(id)
|
|
}
|
|
|
|
// One shared maintenance-and-drain loop.
|
|
wg.Go(func() {
|
|
interval := d.Cfg.Get().PollInterval.Std()
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
d.maintainAndDrain(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
// PollInterval may have been reloaded; reset the ticker so a
|
|
// change takes effect on the next cycle.
|
|
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
|
|
interval = iv
|
|
ticker.Reset(iv)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
wg.Wait()
|
|
return ctx.Err()
|
|
}
|
|
|
|
// pollForIdentity polls one identity's repos via its own forge client,
|
|
// enqueuing any discovered issues under that identity's name. It never
|
|
// drains or reconciles -- those are store-wide and run in the shared
|
|
// maintainAndDrain loop.
|
|
func (d *Daemon) pollForIdentity(ctx context.Context, id *IdentityRunner) {
|
|
if !d.identityActive(ctx, id.ID) {
|
|
return
|
|
}
|
|
d.markPoll()
|
|
cfg := configForIdentity(d.Cfg.Get(), id.Cfg)
|
|
for _, repo := range id.Repos {
|
|
issues := d.pollIssuesWithConfig(ctx, id.Forge, cfg, repo)
|
|
for _, is := range issues {
|
|
labels := strings.Join(is.Labels, ",")
|
|
if d.Tasks != nil {
|
|
d.pollNATS(ctx, id.Forge, cfg, repo, is, labels, string(id.ID))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// maintainAndDrain runs the store-wide passes of a multi-identity cycle:
|
|
// storage cleanup, PR reconciliation, then task dispatch. A single shared
|
|
// loop runs this so the global concurrency cap and per-repo serialization
|
|
// hold across identities.
|
|
func (d *Daemon) maintainAndDrain(ctx context.Context) {
|
|
d.cleanupExpiredStorage(ctx)
|
|
d.reconcilePRs(ctx)
|
|
d.scanPRReviews(ctx)
|
|
d.dispatchBindings(ctx)
|
|
d.drainReactions(ctx)
|
|
if d.Tasks != nil {
|
|
d.drainNATS(ctx)
|
|
}
|
|
}
|
|
|
|
// pollIssuesWithConfig discovers work for one repo using the given forge
|
|
// client and dispatch config. Single-identity mode passes d.Forge/d.Cfg;
|
|
// identity poll loops pass the identity's own forge and configForIdentity,
|
|
// so each identity polls with its own bot user, label, and trigger.
|
|
func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) []forge.Issue {
|
|
switch cfg.Dispatch.Trigger {
|
|
case "label":
|
|
// Empty-label rule shared with workintake.MatchesDispatch: a label
|
|
// trigger needs a non-empty label or it would match every open issue.
|
|
if workintake.RequiresLabel(cfg.Dispatch.Trigger) && cfg.Label == "" {
|
|
d.Log.Error("label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName())
|
|
return nil
|
|
}
|
|
issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
|
|
if err != nil {
|
|
d.Log.Error("label poll failed", "repo", repo.FullName(), "err", err)
|
|
return nil
|
|
}
|
|
return issues
|
|
case "either":
|
|
return d.pollEitherWithConfig(ctx, fg, cfg, repo)
|
|
default: // "assignee" (default)
|
|
issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
|
|
if err != nil {
|
|
d.Log.Error("poll failed", "repo", repo.FullName(), "err", err)
|
|
return nil
|
|
}
|
|
return issues
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) pollEitherWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) []forge.Issue {
|
|
seen := map[int]bool{}
|
|
var out []forge.Issue
|
|
|
|
assigned, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
|
|
if err != nil {
|
|
d.Log.Error("assigned poll failed", "repo", repo.FullName(), "err", err)
|
|
} else {
|
|
for _, is := range assigned {
|
|
seen[is.Number] = true
|
|
out = append(out, is)
|
|
}
|
|
}
|
|
if workintake.RequiresLabel(cfg.Dispatch.Trigger) && cfg.Label == "" {
|
|
d.Log.Error("either trigger configured with an empty label; skipping the label poll (an empty label matches every open issue)", "repo", repo.FullName())
|
|
return out
|
|
}
|
|
labelled, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
|
|
if err != nil {
|
|
d.Log.Error("label poll failed", "repo", repo.FullName(), "err", err)
|
|
} else {
|
|
for _, is := range labelled {
|
|
if !seen[is.Number] {
|
|
out = append(out, is)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Cycle is one poll-and-drain pass: enqueue newly assigned issues,
|
|
// reconcile open PRs, run the binding dispatch loop, then process
|
|
// queued tasks concurrently up to containers.max_concurrency.
|
|
func (d *Daemon) Cycle(ctx context.Context) {
|
|
d.cleanupExpiredStorage(ctx)
|
|
d.poll(ctx)
|
|
d.reconcilePRs(ctx)
|
|
d.scanPRReviews(ctx)
|
|
d.dispatchBindings(ctx)
|
|
d.drainReactions(ctx)
|
|
if d.Tasks != nil {
|
|
d.drainNATS(ctx)
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) {
|
|
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
|
|
if ttl <= 0 {
|
|
return
|
|
}
|
|
if d.Storage != nil {
|
|
n, err := d.Storage.CleanupExpired(ctx, ttl)
|
|
if err != nil {
|
|
d.Log.Warn("persistent volume cleanup failed", "err", err)
|
|
}
|
|
if n > 0 {
|
|
d.Log.Info("expired persistent volumes removed", "count", n)
|
|
}
|
|
}
|
|
}
|
|
|
|
// bindingDispatchBatchLimit caps how many undispatched captures a single
|
|
// dispatchBindings pass will fetch from the store. The cap is defensive
|
|
// (the captures table is also pruned by retention/maxEvents) and bounds a
|
|
// single cycle's worst-case latency on a backlog.
|
|
const bindingDispatchBatchLimit = 100
|
|
|
|
// dispatchBindings walks identified captures on sources with an armed
|
|
// binding and offers each capture to every armed binding on its source.
|
|
// Each binding dispatches it at most once (dispatchOneBinding).
|
|
//
|
|
// Nil Bindings, BindingDispatcher or BindingTaskCreator disables the loop.
|
|
func (d *Daemon) dispatchBindings(ctx context.Context) {
|
|
if d.Bindings == nil || d.BindingDispatcher == nil || d.BindingTaskCreator == nil {
|
|
return
|
|
}
|
|
|
|
// Gather armed sources first so the captures query filters down to
|
|
// only relevant rows.
|
|
bindings, err := d.Bindings.ListBindings(ctx)
|
|
if err != nil {
|
|
d.Log.Warn("list bindings failed", "error", err)
|
|
return
|
|
}
|
|
sources := make([]string, 0, len(bindings))
|
|
for _, b := range bindings {
|
|
if b.Status == binding.StatusArmed && b.Matcher.Source != "" {
|
|
sources = append(sources, b.Matcher.Source)
|
|
}
|
|
}
|
|
if len(sources) == 0 {
|
|
return
|
|
}
|
|
|
|
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, sources, bindingDispatchBatchLimit)
|
|
if err != nil {
|
|
d.Log.Warn("list undispatched captures failed", "error", err)
|
|
return
|
|
}
|
|
workflows, err := d.activeWorkflows(ctx)
|
|
if err != nil {
|
|
d.Log.Warn("binding dispatch: workflow definitions unavailable", "error", err)
|
|
return
|
|
}
|
|
enablement, err := d.workflowEnablement(ctx)
|
|
if err != nil {
|
|
d.Log.Warn("binding dispatch: workflow enablement unavailable", "error", err)
|
|
return
|
|
}
|
|
for _, c := range captures {
|
|
d.dispatchCapture(ctx, c, workflows, enablement)
|
|
}
|
|
}
|
|
|
|
// dispatchCapture offers one capture to every armed binding on its source. A
|
|
// disabled workflow's binding is skipped like an unarmed one, so the capture
|
|
// waits rather than recording a failure every cycle.
|
|
func (d *Daemon) dispatchCapture(ctx context.Context, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection, enablement workflowtask.WorkflowEnablement) {
|
|
armed, err := d.BindingDispatcher.ArmedBindingsForSource(ctx, c.Source)
|
|
if err != nil {
|
|
d.Log.Warn("armed bindings lookup", "source", c.Source, "error", err)
|
|
return
|
|
}
|
|
for _, b := range armed {
|
|
if b.Matcher.Matches(c.Source, c.Dispatchable()) && enablement.Enabled(b.OrgID, b.Workflow) {
|
|
d.dispatchOneBinding(ctx, b, c, workflows)
|
|
}
|
|
}
|
|
}
|
|
|
|
// bindingMapping returns b's mapping, or nil after logging why it is
|
|
// unavailable.
|
|
func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding) *mapping.Mapping {
|
|
if d.Mappings == nil {
|
|
d.Log.Warn("binding dispatch: mapping store unavailable", "binding", b.ID)
|
|
return nil
|
|
}
|
|
m, err := d.Mappings.GetMapping(ctx, b.MappingID)
|
|
if err != nil {
|
|
d.Log.Warn("binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err)
|
|
return nil
|
|
}
|
|
if m == nil {
|
|
d.Log.Warn("binding dispatch: mapping missing", "binding", b.ID, "mapping", b.MappingID)
|
|
}
|
|
return m
|
|
}
|
|
|
|
// dispatchOneBinding offers one capture to one binding. The binding applies
|
|
// when its mapping belongs to the capture's event type. The mapping resolves
|
|
// the payload (a required-field failure records binding_dispatch_failure and
|
|
// stops), counts the match, and the binding's filter must admit the resolved
|
|
// parameters. The workflow it targets must accept the binding's inputs and
|
|
// repository (resolveBindingTarget). The dispatch is then claimed in the binding_dispatches ledger
|
|
// before the task is enqueued: a capture stays listed until every binding for
|
|
// its event type has dispatched it, so the claim is what stops a binding that
|
|
// already fired from firing again on a later cycle. A failed enqueue after the
|
|
// claim loses that dispatch, which is the at-most-once side of the trade.
|
|
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) {
|
|
m := d.bindingMapping(ctx, b)
|
|
if m == nil || m.EventTypeID != c.EventType {
|
|
return
|
|
}
|
|
values, failures := mapping.Resolve(m.Fields, []byte(c.Body))
|
|
if hasBlockingFailure(m.Fields, failures) {
|
|
d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures})
|
|
return
|
|
}
|
|
if d.MappingMatches != nil {
|
|
if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil {
|
|
d.Log.Warn("binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err)
|
|
}
|
|
}
|
|
filter, err := binding.CompileFilter(b.Filter, m.Fields)
|
|
if err != nil {
|
|
d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()})
|
|
return
|
|
}
|
|
if admitted, _ := filter.Admits(values); !admitted {
|
|
return
|
|
}
|
|
|
|
target, reason, ok := d.resolveBindingTarget(b, values, workflows)
|
|
if reason != "" {
|
|
d.recordDispatchFailure(ctx, b, c, reason, nil)
|
|
}
|
|
if !ok {
|
|
return
|
|
}
|
|
// The chain decides dispatch: the workflow's identity may `run` this
|
|
// workflow in its workspace, and the event's signature result and
|
|
// address travel with the request (docs/prds/orgs-and-access.md,
|
|
// "Access decisions"). A denial is recorded and the dispatch is not
|
|
// claimed: the capture stays listed for the next cycle.
|
|
if !d.authorizeDispatch(ctx, b, c, target) {
|
|
return
|
|
}
|
|
d.claimAndEnqueue(ctx, b, c, target, values)
|
|
}
|
|
|
|
// claimAndEnqueue claims the dispatch in the at-most-once ledger and enqueues
|
|
// the task. A failed enqueue after the claim loses that dispatch, which is
|
|
// the at-most-once side of the trade (docs/prds/playbook-binding.md).
|
|
func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) {
|
|
if err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0); err != nil {
|
|
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
|
|
d.Log.Warn("binding dispatch: record", "binding", b.ID, "capture", c.ID, "error", err)
|
|
}
|
|
return
|
|
}
|
|
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
|
|
body := renderBindingBody(values, c)
|
|
if target.declared {
|
|
body = fmt.Sprintf("Started by binding %q from capture %q; the workflow's inputs travel with the task.", b.Name, c.ID)
|
|
}
|
|
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, "", b.ID, b.Version, target.inputs)
|
|
if err != nil {
|
|
d.Log.Warn("binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
|
|
return
|
|
}
|
|
if c.Unsigned {
|
|
d.markUnsignedStart(ctx, task.ID, b, c)
|
|
}
|
|
}
|
|
|
|
// authorizeDispatch evaluates the policy chain for the workflow's identity
|
|
// before the dispatch is claimed: the identity may `run` the workflow in its
|
|
// workspace, with the event's signature result and source address as context.
|
|
// A denial is recorded with the level that decided it and the deciding
|
|
// policies, and reported as a binding_dispatch_failure.
|
|
func (d *Daemon) authorizeDispatch(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget) bool {
|
|
if d.Access == nil || d.Principals == nil {
|
|
return true
|
|
}
|
|
principal, err := d.Principals.PrincipalFor(ctx, d.RootIdentityID)
|
|
if err != nil {
|
|
d.Log.Warn("binding dispatch: principal unavailable; dispatch parked for the next cycle",
|
|
"binding", b.ID, "capture", c.ID, "error", err)
|
|
return false
|
|
}
|
|
resource := access.Resource{
|
|
Kind: access.KindWorkflow, ID: b.Workflow, Org: principal.Org,
|
|
}
|
|
contextValue := access.Context{Signature: signatureOf(c), Addr: c.RemoteAddr}
|
|
decision := d.Access.Authorize(principal, access.ActionRun, resource, contextValue)
|
|
if decision.Allowed {
|
|
return true
|
|
}
|
|
if d.Denials != nil {
|
|
denial := access.Denial{
|
|
Principal: principal.IdentityID, Org: principal.Org, Action: access.ActionRun,
|
|
Kind: resource.Kind, ResourceID: resource.ID, Level: decision.Level,
|
|
Policies: decision.Policies,
|
|
}
|
|
if err := d.Denials.RecordDenial(ctx, denial); err != nil {
|
|
d.Log.Warn("binding dispatch: record denial", "binding", b.ID, "error", err)
|
|
}
|
|
}
|
|
d.recordDispatchFailure(ctx, b, c, "access denied", map[string]any{
|
|
"level": string(decision.Level), "policies": decision.Policies,
|
|
})
|
|
return false
|
|
}
|
|
|
|
// signatureOf names the event's signature result as the dispatch context
|
|
// carries it: a signed event and an approved unsigned event are the two
|
|
// dispatchable shapes (storecontract.CapturedEvent.Dispatchable).
|
|
func signatureOf(c storecontract.CapturedEvent) string {
|
|
switch {
|
|
case c.Authenticated:
|
|
return "valid"
|
|
case c.Unsigned:
|
|
return "unsigned"
|
|
}
|
|
return "unverified"
|
|
}
|
|
|
|
// markUnsignedStart records on a task's timeline that an unsigned event
|
|
// started it.
|
|
func (d *Daemon) markUnsignedStart(ctx context.Context, taskID int64, b binding.Binding, c storecontract.CapturedEvent) {
|
|
if _, err := d.Store.InsertEvent(ctx, events.Event{
|
|
TaskID: taskID,
|
|
Kind: events.KindUnsignedEvent,
|
|
Detail: fmt.Sprintf("started by an unsigned event from source %q", c.Source),
|
|
Data: map[string]any{"source": c.Source, "capture_id": c.ID, "binding_id": b.ID},
|
|
}); err != nil {
|
|
d.Log.Warn("binding dispatch: unsigned marker", "task", taskID, "error", err)
|
|
}
|
|
}
|
|
|
|
// recordDispatchFailure records why a binding did not dispatch a capture.
|
|
func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) {
|
|
data := map[string]any{
|
|
"binding_id": b.ID,
|
|
"binding_version": b.Version,
|
|
"capture_id": c.ID,
|
|
}
|
|
maps.Copy(data, extra)
|
|
_, _ = d.Store.InsertEvent(ctx, events.Event{
|
|
Kind: "binding_dispatch_failure",
|
|
Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason),
|
|
Data: data,
|
|
})
|
|
}
|
|
|
|
// hasBlockingFailure reports whether any of the failures belongs to a
|
|
// required field. The mapping package reports failures only by field
|
|
// name and reason -- the Field's Required flag is on the Field
|
|
// declaration, so the dispatch loop re-derives it from the full
|
|
// Field list. Failures on optional fields are warnings, not blocks:
|
|
// the workflow still receives a partial value map.
|
|
func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool {
|
|
if len(failures) == 0 {
|
|
return false
|
|
}
|
|
required := make(map[string]bool, len(fields))
|
|
for _, f := range fields {
|
|
if f.Required {
|
|
required[f.Name] = true
|
|
}
|
|
}
|
|
for _, fail := range failures {
|
|
if required[fail.FieldName] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// renderBindingBody builds the task body from the values a mapping
|
|
// resolved, one "key=value" pair per line. Empty maps still produce
|
|
// a body that records the source capture id so the workflow has
|
|
// provenance to hand back to the operator.
|
|
func renderBindingBody(values map[string]any, c storecontract.CapturedEvent) string {
|
|
if len(values) == 0 {
|
|
return fmt.Sprintf("(no fields resolved from capture %q)", c.ID)
|
|
}
|
|
parts := make([]string, 0, len(values))
|
|
for k, v := range values {
|
|
parts = append(parts, fmt.Sprintf("%s=%v", k, v))
|
|
}
|
|
return strings.Join(parts, "\n")
|
|
}
|
|
|
|
// resolveBindingRepo picks the owner/repo a binding-spawned task
|
|
// operates against. A binding that pins Owner/Repo (binding.Binding's
|
|
// Owner/Repo fields, validated as a complete pair or not set at all)
|
|
// dispatches to that target unconditionally -- multi-repo deployments
|
|
// use this to disambiguate. An unpinned binding falls back to the
|
|
// single-configured-repo behaviour that predates the pin: with exactly
|
|
// one configured repo the choice is unambiguous; with zero or more than
|
|
// one the loop refuses to dispatch (logged below) so a binding cannot
|
|
// silently fire against the wrong target.
|
|
func (d *Daemon) resolveBindingRepo(b binding.Binding) (string, string, bool) {
|
|
if b.Owner != "" && b.Repo != "" {
|
|
return b.Owner, b.Repo, true
|
|
}
|
|
repos := d.Cfg.Get().Repos
|
|
if len(repos) == 1 {
|
|
return repos[0].Owner, repos[0].Name, true
|
|
}
|
|
d.Log.Warn("binding dispatch: cannot resolve target repo",
|
|
"hint", "configure exactly one [[repos]] entry, or pin the binding to a specific owner/repo",
|
|
"configured_repos", len(repos))
|
|
return "", "", false
|
|
}
|
|
|
|
// drainReactions turns available review reactions into queued remediate
|
|
// runs. It runs before the task drain so a reaction queued in this pass can
|
|
// be claimed by it; the consumer is built lazily once, so its drop counter
|
|
// spans the process lifetime.
|
|
func (d *Daemon) drainReactions(ctx context.Context) {
|
|
if d.Reactions == nil || d.Store == nil {
|
|
return
|
|
}
|
|
if d.reactionConsumer == nil {
|
|
d.reactionConsumer = newReactionConsumer(
|
|
// The store's owned-PR lookup is the authorization boundary: a
|
|
// reaction only ever causes work against a PR archie opened and
|
|
// still owns (pr-review-remediation.md decision 3).
|
|
d.Store,
|
|
d.Store,
|
|
d.botUserForTask,
|
|
d.Log,
|
|
)
|
|
}
|
|
if _, err := d.reactionConsumer.drain(ctx, d.Reactions, maxReactionsPerCycle); err != nil {
|
|
d.Log.Error("reaction drain failed", "err", err)
|
|
}
|
|
}
|
|
|
|
// botUserForTask resolves the bot login that owns task, so the reaction
|
|
// consumer can exclude Archie's own comments. Identity-derived for
|
|
// multi-identity deployments, the global forge bot user otherwise.
|
|
func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
|
|
if runner := d.identityFor(task); runner != nil {
|
|
return runner.Cfg.BotUser
|
|
}
|
|
return d.Cfg.Get().BotUser
|
|
}
|
|
|
|
// drainNATS processes tasks from NATS, falling back to the task store's ClaimNext for
|
|
// requeued tasks (waiting_human approval, retry-parked) that didn't come
|
|
// through a NATS publish.
|
|
func (d *Daemon) drainNATS(ctx context.Context) {
|
|
dispatcher := newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask)
|
|
for ctx.Err() == nil {
|
|
msg, err := d.Tasks.Fetch(ctx)
|
|
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
|
|
d.Log.Error("nats fetch failed", "err", err)
|
|
break
|
|
}
|
|
if err == nil {
|
|
d.submitNATSTask(ctx, dispatcher, msg)
|
|
continue
|
|
}
|
|
|
|
// ErrNoMessage means the stream is drained. That is precisely when
|
|
// the SQLite fallback must run: requeued tasks (waiting_human
|
|
// approval, retry-parked) never went through a NATS publish, so
|
|
// nothing will ever arrive on the stream to trigger them.
|
|
task, claimErr := d.Store.ClaimNext(ctx)
|
|
if claimErr != nil {
|
|
d.Log.Error("sqlite claim failed", "err", claimErr)
|
|
break
|
|
}
|
|
if task == nil {
|
|
break
|
|
}
|
|
dispatcher.Submit(ctx, task, d.process)
|
|
}
|
|
dispatcher.Wait()
|
|
}
|
|
|
|
// submitNATSTask decodes a fetched message and queues it for processing. A
|
|
// message that cannot be decoded is acked rather than redelivered: it will
|
|
// never decode on a retry, so leaving it unacked would block the queue.
|
|
func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) {
|
|
envelope, err := workintake.DecodeTask(msg.Data())
|
|
if err != nil {
|
|
d.Log.Error("nats decode failed", "subject", msg.Subject(), "err", err)
|
|
_ = msg.Ack()
|
|
return
|
|
}
|
|
task := &workflow.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
|
|
dispatcher.Submit(ctx, task, func(ctx context.Context, _ *workflow.Task) {
|
|
d.processNATSTask(ctx, msg)
|
|
})
|
|
}
|
|
|
|
// taskDispatcher bounds task execution globally while preserving the default
|
|
// one-task-per-repository safety rule. Submit order is retained within a repo;
|
|
// tasks for different repos may run concurrently. Repos for which
|
|
// allowConcurrent reports true opt out of the same-repo serialization
|
|
// entirely -- the global slot limit is the only bound on their concurrency.
|
|
type taskDispatcher struct {
|
|
slots chan struct{}
|
|
allowConcurrent func(task *workflow.Task) bool
|
|
|
|
mu sync.Mutex
|
|
repoTail map[string]chan struct{}
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *workflow.Task) bool) *taskDispatcher {
|
|
var slots chan struct{}
|
|
if maxConcurrency > 0 {
|
|
slots = make(chan struct{}, maxConcurrency)
|
|
}
|
|
if allowConcurrent == nil {
|
|
allowConcurrent = func(*workflow.Task) bool { return false }
|
|
}
|
|
return &taskDispatcher{
|
|
slots: slots,
|
|
allowConcurrent: allowConcurrent,
|
|
repoTail: make(map[string]chan struct{}),
|
|
}
|
|
}
|
|
|
|
func (d *taskDispatcher) Submit(
|
|
ctx context.Context,
|
|
task *workflow.Task,
|
|
process func(context.Context, *workflow.Task),
|
|
) {
|
|
repo := task.Owner + "/" + task.Repo
|
|
|
|
var previous, done chan struct{}
|
|
if !d.allowConcurrent(task) {
|
|
done = make(chan struct{})
|
|
d.mu.Lock()
|
|
previous = d.repoTail[repo]
|
|
d.repoTail[repo] = done
|
|
d.mu.Unlock()
|
|
}
|
|
|
|
d.wg.Go(func() {
|
|
if previous != nil {
|
|
<-previous
|
|
}
|
|
if d.slots != nil {
|
|
d.slots <- struct{}{}
|
|
defer func() { <-d.slots }()
|
|
}
|
|
if done != nil {
|
|
defer func() {
|
|
close(done)
|
|
d.mu.Lock()
|
|
if d.repoTail[repo] == done {
|
|
delete(d.repoTail, repo)
|
|
}
|
|
d.mu.Unlock()
|
|
}()
|
|
}
|
|
process(ctx, task)
|
|
})
|
|
}
|
|
|
|
func (d *taskDispatcher) Wait() {
|
|
d.wg.Wait()
|
|
}
|
|
|
|
func (d *Daemon) poll(ctx context.Context) {
|
|
if !d.identityActive(ctx, d.RootIdentityID) {
|
|
return
|
|
}
|
|
d.markPoll()
|
|
for _, repo := range d.Cfg.Get().Repos {
|
|
issues := d.pollIssues(ctx, repo)
|
|
for _, is := range issues {
|
|
labels := strings.Join(is.Labels, ",")
|
|
if d.Tasks != nil {
|
|
d.pollNATS(ctx, d.Forge, d.Cfg.Get(), repo, is, labels, "")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// pollNATS publishes discovered issues to NATS (new flow).
|
|
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) {
|
|
// Read-only existence check prevents rediscovering an existing task.
|
|
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
|
|
if err != nil {
|
|
d.Log.Error("task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
|
return
|
|
}
|
|
if existing != nil {
|
|
return
|
|
}
|
|
// Classify and encode here: the envelope, its routing kind and its
|
|
// subject belong to the work-intake domain, not to the transport.
|
|
parsed := workintake.SplitLabels(labels)
|
|
task := workintake.TaskEnvelope{
|
|
Owner: repo.Owner,
|
|
Repo: repo.Name,
|
|
Number: is.Number,
|
|
Title: is.Title,
|
|
Body: is.Body,
|
|
Labels: parsed,
|
|
Identity: identity,
|
|
Kind: workintake.KindForLabels(parsed),
|
|
}
|
|
if err := d.PublishTask(ctx, task); err != nil {
|
|
d.Log.Error("task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
|
return
|
|
}
|
|
d.acknowledge(ctx, fg, cfg, repo, is)
|
|
}
|
|
|
|
// PublishTask validates, resolves the producing identity's org, encodes and
|
|
// publishes a task envelope. The idempotency key means rediscovering the same
|
|
// issue on a later poll does not enqueue the work twice. It is the single
|
|
// enqueue path shared by the poller and the forge webhook intake, so neither
|
|
// can drift in how a discovered issue becomes a task -- including the org the
|
|
// key is built from (docs/prds/orgs-and-access.md, "Events and task
|
|
// identity").
|
|
func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
|
|
if d.Tasks == nil {
|
|
return fmt.Errorf("publish task %s: no task bus configured", task.Ref())
|
|
}
|
|
if err := task.Kind.Validate(); err != nil {
|
|
return fmt.Errorf("publish task %s: %w", task.Ref(), err)
|
|
}
|
|
resolved, err := d.identityOrg(ctx, d.publisherIdentity(task.Identity))
|
|
if err != nil {
|
|
return fmt.Errorf("publish task %s: resolve org: %w", task.Ref(), err)
|
|
}
|
|
task.Org = resolved
|
|
payload, err := task.Encode()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
|
|
}
|
|
|
|
// PublishReaction stamps the resolved org on a reaction and publishes it on
|
|
// the reaction subject. A reaction the review scan already stamped with its
|
|
// task's org keeps it; otherwise (the webhook producer, single-identity) the
|
|
// root identity's org is used, so a webhook and a poll delivery of one review
|
|
// key the same. It is the single reaction write path shared by the scan and
|
|
// the webhook receiver.
|
|
func (d *Daemon) PublishReaction(ctx context.Context, reaction workintake.ReviewCommentEnvelope) error {
|
|
if d.Tasks == nil {
|
|
return fmt.Errorf("publish reaction %s: no task bus configured", reaction.Ref())
|
|
}
|
|
if reaction.Org == "" {
|
|
resolved, err := d.identityOrg(ctx, d.RootIdentityID)
|
|
if err != nil {
|
|
return fmt.Errorf("publish reaction %s: resolve org: %w", reaction.Ref(), err)
|
|
}
|
|
reaction.Org = resolved
|
|
}
|
|
payload, err := reaction.Encode()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return d.Tasks.PublishUnique(ctx, reaction.Subject(), reaction.IdempotencyKey(), payload)
|
|
}
|
|
|
|
// publisherIdentity is the identity a producer publishes as: the envelope's
|
|
// identity when it names one (an ID, or a name for a hand-built envelope),
|
|
// otherwise the root identity of a single-identity install.
|
|
func (d *Daemon) publisherIdentity(ref string) identity.IdentityID {
|
|
if ref == "" {
|
|
return d.RootIdentityID
|
|
}
|
|
for _, runner := range d.Identities {
|
|
if runner == nil {
|
|
continue
|
|
}
|
|
if runner.Name == ref || string(runner.ID) == ref {
|
|
return runner.ID
|
|
}
|
|
}
|
|
return identity.IdentityID(ref)
|
|
}
|
|
|
|
// identityOrg resolves the org an identity serves through the State Store
|
|
// before a producer publishes. An unwired PrincipalSource (an install with no
|
|
// State Store) resolves to the default org; a store failure is returned so the
|
|
// producer retries rather than publishing under a guessed org, which would key
|
|
// the same issue differently on a later delivery.
|
|
func (d *Daemon) identityOrg(ctx context.Context, id identity.IdentityID) (org.OrgID, error) {
|
|
if d.Principals == nil {
|
|
return org.DefaultOrgID, nil
|
|
}
|
|
principal, err := d.Principals.PrincipalFor(ctx, id)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return principal.Org, nil
|
|
}
|
|
|
|
// acknowledge posts the pickup reaction and queued event
|
|
// using fg -- the forge client that owns repo (identity-scoped or root) --
|
|
// and the ack reaction from cfg, the dispatch config that owns this poll.
|
|
func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) {
|
|
d.Log.Info("issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title)
|
|
if ack := cfg.Dispatch.AckReaction; ack != "" {
|
|
if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil {
|
|
d.Log.Warn("ack reaction failed", "issue", is.Number, "err", err)
|
|
}
|
|
}
|
|
d.emit(events.Event{
|
|
Kind: events.KindTaskQueued, Repo: repo.FullName(),
|
|
Issue: is.Number, Detail: is.Title,
|
|
})
|
|
}
|
|
|
|
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
|
|
// and runs the workflow. The message is ack'd on terminal (park is a valid
|
|
// outcome). Nak on transient errors so NATS redelivers.
|
|
func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message) {
|
|
tm, err := workintake.DecodeTask(msg.Data())
|
|
if err != nil {
|
|
d.Log.Error("nats decode failed", "err", err)
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
// bad message, don't retry
|
|
return
|
|
}
|
|
|
|
if ok, reason, _ := d.identityMayAct(ctx, tm.Identity); !ok {
|
|
d.Log.Error("nats intake rejected: "+reason, "subject", msg.Subject())
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
|
|
if err != nil {
|
|
d.Log.Error("nats enqueue failed", "err", err)
|
|
if err := msg.Nak(); err != nil {
|
|
d.Log.Warn("nats nak failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
if !inserted {
|
|
// Already tracked -- dedup. Ack and move on.
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
|
|
if err != nil {
|
|
d.Log.Error("nats claim failed", "err", err)
|
|
if err := msg.Nak(); err != nil {
|
|
d.Log.Warn("nats nak failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
if task == nil {
|
|
// Claimed by another consumer, or task is not queued. Ack.
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
d.process(ctx, task)
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
}
|
|
|
|
// pollIssues discovers work for one repo according to the root dispatch
|
|
// config, using the root forge client. Identity poll loops use
|
|
// pollIssuesWithConfig with their own forge and config instead.
|
|
func (d *Daemon) pollIssues(ctx context.Context, repo config.Repo) []forge.Issue {
|
|
return d.pollIssuesWithConfig(ctx, d.Forge, d.Cfg.Get(), repo)
|
|
}
|
|
|
|
// closeResolvedIssue closes the forge issue behind a finished task.
|
|
//
|
|
// A chat task's issue number is synthetic and matches no forge issue, so it
|
|
// is skipped. Failure is logged rather than returned: the task's own state is
|
|
// already correct, and the next reconcile pass will not retry, so a warning
|
|
// is the honest outcome -- the issue simply stays open.
|
|
func (d *Daemon) closeResolvedIssue(ctx context.Context, fg forge.Forge, task *workflow.Task, comment string) {
|
|
if !task.IsForgeBacked() {
|
|
return
|
|
}
|
|
if err := fg.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil {
|
|
d.Log.Warn("closing resolved issue failed; it stays open and will be re-polled",
|
|
"repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err)
|
|
}
|
|
}
|
|
|
|
// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state
|
|
// and cleans up their worktrees.
|
|
func (d *Daemon) reconcilePRs(ctx context.Context) {
|
|
tasks, err := d.Store.OpenPRs(ctx)
|
|
if err != nil {
|
|
d.Log.Error("reconcile query failed", "err", err)
|
|
return
|
|
}
|
|
for _, t := range tasks {
|
|
fg := d.forgeFor(&t)
|
|
trees := d.treesFor(&t)
|
|
// PRState/PR reconciliation is valid regardless of Source -- a
|
|
// chat-sourced task can still open a real PR against a real
|
|
// branch. Only the issue-label call (tied to the synthetic
|
|
// issue number) is forge-only.
|
|
state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber)
|
|
if err != nil {
|
|
d.Log.Warn("PR state check failed", "pr", t.PRNumber, "err", err)
|
|
continue
|
|
}
|
|
// OpenPRs returns a deliberately narrow projection of a pr_open task
|
|
// (id/owner/repo/issue/pr/status/source/identity) that does not carry
|
|
// the attempt, so the outcome event below reads it from the task's own
|
|
// row. A failed lookup costs attribution and never the merge handling:
|
|
// the status transition and the worktree cleanup still run. OpenPRs
|
|
// carries attempt, so these events are attributed without a second
|
|
// read per open PR on every reconcile tick.
|
|
attempt := t.Attempt
|
|
switch state {
|
|
case "merged":
|
|
_ = d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusMerged, "")
|
|
_ = trees.Cleanup(t.Owner, t.Repo, t.IssueNumber)
|
|
// Close the issue ourselves rather than relying on the forge
|
|
// noticing a "Closes #N" in the PR body. Nothing else closes it:
|
|
// LinkBranch is sidebar linkage on Gitea and a no-op on GitHub.
|
|
// Left open the issue stays labelled and assigned, the next poll
|
|
// re-enqueues it, and once the dashboard's Clear removes the task
|
|
// row that is a second implementation and a second PR for work
|
|
// already merged.
|
|
d.closeResolvedIssue(ctx, fg, &t, fmt.Sprintf(
|
|
"Closed by #%d, merged by archie.", t.PRNumber,
|
|
))
|
|
d.Log.Info("PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
|
|
d.emit(events.Event{
|
|
Kind: events.KindPRMerged, TaskID: t.ID, Attempt: attempt,
|
|
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
|
|
Data: map[string]any{"pr": t.PRNumber},
|
|
})
|
|
case "closed":
|
|
_ = d.Store.Transition(ctx, t.ID, workflow.StatusPROpen, workflow.StatusRejected, "PR closed without merge")
|
|
_ = trees.Cleanup(t.Owner, t.Repo, t.IssueNumber)
|
|
d.Log.Info("PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
|
|
d.emit(events.Event{
|
|
Kind: events.KindPRRejected, TaskID: t.ID, Attempt: attempt,
|
|
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
|
|
Data: map[string]any{"pr": t.PRNumber},
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) process(ctx context.Context, task *workflow.Task) {
|
|
// Register before any work starts so the task is stoppable for its
|
|
// whole life, including the slow setup -- clone, worktree prepare,
|
|
// image pull -- which is exactly when someone realises they asked for
|
|
// the wrong thing. Every caller reaches execution through here, so
|
|
// this is the one place that has to do it.
|
|
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
|
defer finished()
|
|
defer d.openTaskLog(task)()
|
|
|
|
if d.parkIfIdentityUnresolvable(ctx, task) {
|
|
return
|
|
}
|
|
|
|
trees := d.treesFor(task)
|
|
repo, ok := d.repoFor(task)
|
|
if !ok && task.HasRepository() {
|
|
d.parkRunningTask(ctx, task.ID, "repo no longer in config", taskstate.ParkTerminal)
|
|
return
|
|
}
|
|
if d.ContainerPool == nil || d.Tasks == nil {
|
|
d.parkCapabilityUnavailable(ctx, task, repo)
|
|
return
|
|
}
|
|
|
|
d.Log.Info("processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
|
|
workDir, ok := d.prepareWorkspace(ctx, task, trees, repo)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !task.HasRepository() {
|
|
defer func() {
|
|
if err := trees.RemoveScratch(task.ID); err != nil {
|
|
d.Log.Warn("scratch workspace cleanup failed", "task", task.ID, "err", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
// The workflow is pinned before the container starts, because the
|
|
// profile it names decides the container's image.
|
|
profile, ok := d.pinTaskProfile(ctx, task)
|
|
if !ok {
|
|
return
|
|
}
|
|
if profile.IsKit() {
|
|
d.runKitTask(ctx, task, repo, workDir, profile)
|
|
d.teardownStorage(ctx, task, repo, workDir)
|
|
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
|
return
|
|
}
|
|
ctr, revokeStateStoreGrant, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile.Image)
|
|
if !ok {
|
|
return
|
|
}
|
|
defer d.ContainerPool.Release(ctx, ctr)
|
|
defer revokeStateStoreGrant()
|
|
|
|
// Hand the whole task to archie-agent in one NATS round trip. archie-agent
|
|
// proxies Store/Forge/worktree-push calls back to archied over storerpc/
|
|
// forgerpc/worktreerpc -- by the time runViaAgent returns, the task's
|
|
// terminal state already landed via those calls. The task's identity
|
|
// travels in the taskrun request; the agent scopes its forgerpc and
|
|
// worktreerpc clients to that identity, and the daemon registered one
|
|
// server pair per identity (plus the root pair), so a container-mode
|
|
// task is always served by its own forge client and worktree manager.
|
|
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
|
runCtx, stopWatch := withContainerExit(limitCtx, ctr.Exited())
|
|
d.runViaAgent(runCtx, task, repo, profile, nil)
|
|
stopWatch()
|
|
stopLimit()
|
|
|
|
d.teardownStorage(ctx, task, repo, workDir)
|
|
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
|
}
|
|
|
|
// teardownStorage runs after the workflow completes. The Docker backend is
|
|
// a no-op; future backends (temp volumes, NFS leases) use this hook.
|
|
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
|
|
if d.Storage != nil {
|
|
_ = d.Storage.Teardown(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
})
|
|
}
|
|
}
|
|
|
|
// prepareWorkspace makes the directory the container binds as its
|
|
// workspace, which must exist before Acquire: a fresh clone of the task's
|
|
// repository, or an empty scratch directory for a task with none. It parks
|
|
// the task and reports false when it cannot.
|
|
func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) {
|
|
if !task.HasRepository() {
|
|
dir, err := trees.PrepareScratch(task.ID)
|
|
if err != nil {
|
|
d.Log.Error("scratch workspace prepare failed", "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient)
|
|
return "", false
|
|
}
|
|
return dir, true
|
|
}
|
|
// The retry path selects the worktree mode from the task row. A task that
|
|
// continues an already-open PR branch -- the remediate workflow -- resumes
|
|
// that branch's remote tip instead of resetting onto base: preparing one
|
|
// the ordinary way discarded the commits the implement run had already
|
|
// pushed, so the agent remediated a PR from a tree that no longer contained
|
|
// it. Every other task starts fresh on base. This runs before Acquire,
|
|
// because the daemon holds the forge credential that fetches the branch and
|
|
// the container must find the PR's work already in place. It also runs on a
|
|
// retry whose local worktree is gone: Prepare clones a missing worktree for
|
|
// either target, which is how a resume reaches a task whose clone the
|
|
// terminal cleanup removed.
|
|
target := worktree.Fresh
|
|
if continuesOpenPRBranch(task) {
|
|
if task.Branch == "" {
|
|
// A remediation task is queued against the branch its implement
|
|
// run pushed, so an empty branch is a dispatch bug upstream and no
|
|
// retry can fix it.
|
|
d.Log.Error("remediate task has no persisted branch", "task", task.ID)
|
|
d.parkRunningTask(ctx, task.ID, "worktree resume failed: remediate task has no branch to resume", taskstate.ParkNeedsHuman)
|
|
return "", false
|
|
}
|
|
target = worktree.Target(task.Branch)
|
|
}
|
|
// Every task gets an independent full clone. The former
|
|
// PreparePersistent path shared objects with a per-repo bare cache;
|
|
// go-git has no --dissociate, so a shared cache would stay a live
|
|
// dependency of each worktree and expiring one would corrupt running
|
|
// tasks. repo.PersistentStorage still governs the container volume.
|
|
dir, branch, err := trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
|
|
if err != nil {
|
|
reason := "worktree prepare failed: " + err.Error()
|
|
if target != worktree.Fresh {
|
|
reason = "worktree resume failed: " + err.Error()
|
|
}
|
|
d.Log.Error("worktree prepare failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, reason, taskstate.ParkTransient)
|
|
return "", false
|
|
}
|
|
task.Branch = branch
|
|
if err := d.Store.Update(ctx, task); err != nil {
|
|
d.Log.Warn("task branch not persisted", "task", task.ID, "err", err)
|
|
}
|
|
return dir, true
|
|
}
|
|
|
|
// continuesOpenPRBranch reports whether the task must continue a branch archie
|
|
// already pushed rather than start from base. Only the remediate workflow does:
|
|
// it addresses review feedback on an open archie-owned PR and reuses that PR's
|
|
// branch (docs/prds/pr-review-remediation.md decision 4), while every other
|
|
// workflow starts fresh work. The workflow is read from the task row, where the
|
|
// reaction consumer wrote it before queueing the run -- a decision about this
|
|
// task that no later routing may overturn.
|
|
func continuesOpenPRBranch(task *workflow.Task) bool {
|
|
return task.Workflow == "remediate"
|
|
}
|
|
|
|
// cleanupTerminalTaskWorktree removes a task's worktree once its forge side is
|
|
// settled, unless the worktree still holds work no commit captured.
|
|
//
|
|
// The cleanliness guard is what keeps agent.run's deliverable: that step ends
|
|
// the workflow StatusCompleted and works in a real clone whose edits it never
|
|
// commits, so a status-only rule deletes the work. Nothing else holds it -- not
|
|
// the branch, not the store. A worktree whose cleanliness cannot be read is
|
|
// kept for the same reason: a wrong keep costs disk, a wrong delete costs the
|
|
// work.
|
|
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
|
if d.Store == nil || trees == nil || task == nil || !task.HasRepository() {
|
|
return
|
|
}
|
|
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
|
|
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
|
|
if err != nil {
|
|
d.Log.Warn("terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
|
|
return
|
|
}
|
|
if latest == nil {
|
|
return
|
|
}
|
|
// Statuses whose forge side is settled, so the worktree holds nothing a
|
|
// later run needs: the PR merged, the PR was rejected, an operator refused
|
|
// the work, or the run finished without opening a PR at all (a no-change
|
|
// build, a triage that needed no code -- StatusCompleted). The PRNumber
|
|
// guard keeps the worktree when a PR exists for the reconciler; the
|
|
// cleanliness guard keeps it when it holds uncaptured work.
|
|
switch latest.Status {
|
|
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted:
|
|
if latest.PRNumber != 0 {
|
|
return
|
|
}
|
|
if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) {
|
|
return
|
|
}
|
|
if err := trees.Cleanup(task.Owner, task.Repo, task.IssueNumber); err != nil {
|
|
d.Log.Warn("terminal worktree cleanup failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has
|
|
// to survive cleanup because it may hold work no commit captured. A worktree
|
|
// that is not on disk at all has nothing to keep, so a task whose clone was
|
|
// already removed still cleans up whatever leftovers share its path.
|
|
func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool {
|
|
dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
|
|
if _, err := os.Stat(dir); err != nil {
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
return false
|
|
}
|
|
d.Log.Warn("terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
|
|
return true
|
|
}
|
|
uncommitted, err := trees.HasUncommittedChanges(ctx, dir)
|
|
if err != nil {
|
|
d.Log.Warn("terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
|
|
return true
|
|
}
|
|
if uncommitted {
|
|
d.Log.Info("terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// openTaskLog opens task's log sink for the lifetime of one process() call
|
|
// and returns the function to close it. That lifetime covers preparation,
|
|
// the full-task container handoff, and teardown. A failure to open is not
|
|
// fatal to the task -- it means this
|
|
// attempt's own output won't be recoverable if something goes wrong, which
|
|
// is worse than the pre-existing state, not equal to it, so the run
|
|
// proceeds rather than parking over a logging problem.
|
|
func (d *Daemon) openTaskLog(task *workflow.Task) func() {
|
|
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
|
|
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
|
|
}
|
|
return func() {
|
|
if err := d.TaskLogs.Close(task.ID); err != nil {
|
|
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// acquireTaskContainer prepares and acquires the sandbox container for a
|
|
// task, reporting false when it could not -- having already parked the
|
|
// task and logged why. The caller releases the container it returns.
|
|
func (d *Daemon) acquireTaskContainer(
|
|
ctx context.Context,
|
|
task *workflow.Task,
|
|
repo config.Repo,
|
|
workDir string,
|
|
image string,
|
|
) (*container.Container, func(), bool) {
|
|
park := func(reason string, err error) {
|
|
d.Log.Error(reason, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error(), taskstate.ParkTransient)
|
|
}
|
|
|
|
// A watched-repositories binding or the operator chat trigger assigns
|
|
// the pull request generically through Task.Inputs (its declared
|
|
// pr_number input); archie's own PRs sets PRNumber directly and never
|
|
// goes through either path. Resolving here, before the brief is
|
|
// written, means task.json and the prefetch below always agree with
|
|
// what every later reader (stagePRIntake, output posting) sees.
|
|
if task.Workflow == "pr-review" {
|
|
task.PRNumber = task.EffectivePRNumber()
|
|
}
|
|
|
|
if err := writeTaskBrief(workDir, task); err != nil {
|
|
park("task.json write failed", err)
|
|
return nil, nil, false
|
|
}
|
|
|
|
// pr-review needs an external pull request's metadata, diff and head
|
|
// snapshot inside the container, but the container must never hold a
|
|
// forge credential (docs/development/agent.md, "Giving the agent
|
|
// something new"). The daemon fetches it here, before the container
|
|
// exists, using its own credentialed forge client, and writes the result
|
|
// into workDir -- the same host directory Docker bind-mounts at
|
|
// storage.WorktreeMountDir -- so the sandboxed pipeline can read it back
|
|
// with prsource.MountSource and no network call of its own.
|
|
if task.Workflow == "pr-review" {
|
|
if err := prefetchPRReview(ctx, d.forgeFor(task), task.Owner, task.Repo, task.PRNumber, workDir); err != nil {
|
|
park("pr-review prefetch failed", err)
|
|
return nil, nil, false
|
|
}
|
|
}
|
|
|
|
// Guard: Storage may be nil if the daemon was wired incorrectly. In
|
|
// normal operation, Storage is always set when ContainerPool is set.
|
|
if d.Storage == nil {
|
|
d.Log.Error("storage backend is nil -- cannot acquire container")
|
|
d.parkRunningTask(ctx, task.ID, "storage backend not configured", taskstate.ParkTransient)
|
|
return nil, nil, false
|
|
}
|
|
|
|
mounts, err := d.Storage.Setup(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
})
|
|
if err != nil {
|
|
park("storage setup failed", err)
|
|
return nil, nil, false
|
|
}
|
|
|
|
stateStoreToken, revokeStateStoreGrant, err := d.stateStoreGrantToken(task)
|
|
if err != nil {
|
|
if terr := d.Storage.Teardown(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
}); terr != nil {
|
|
d.Log.Warn("storage teardown after state store grant failure failed", "err", terr)
|
|
}
|
|
park("state store grant failed", err)
|
|
return nil, nil, false
|
|
}
|
|
|
|
ctr, err := d.ContainerPool.Acquire(ctx, image, mounts, d.containerEnv(task, stateStoreToken))
|
|
if err != nil {
|
|
revokeStateStoreGrant()
|
|
// Roll back the storage we just set up: the mounts were created
|
|
// for this task but no container will use them. Leaking them until
|
|
// a later TTL sweep is not acceptable on a backend that allocates
|
|
// real resources (volumes, NFS leases).
|
|
if terr := d.Storage.Teardown(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
}); terr != nil {
|
|
d.Log.Warn("storage teardown after acquire failure failed", "err", terr)
|
|
}
|
|
park("container acquire failed", err)
|
|
return nil, nil, false
|
|
}
|
|
return ctr, revokeStateStoreGrant, true
|
|
}
|
|
|
|
// writeTaskBrief writes task.json, the container's boot-time brief.
|
|
func writeTaskBrief(workDir string, task *workflow.Task) error {
|
|
return container.WriteTaskJSON(workDir, container.TaskPayload{
|
|
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
|
|
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
|
|
Labels: strings.Split(task.Labels, ","),
|
|
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
|
|
Inputs: task.Inputs,
|
|
})
|
|
}
|
|
|
|
// parkCapabilityUnavailable parks a task whose daemon-side capability is
|
|
// missing (container pool or task transport): both are environmental, so
|
|
// the park class is transient and the KindParked event carries the class
|
|
// for the dashboard. The reason distinguishes the two in logs and events.
|
|
func (d *Daemon) parkCapabilityUnavailable(ctx context.Context, task *workflow.Task, repo config.Repo) {
|
|
var reason string
|
|
if d.ContainerPool == nil {
|
|
reason = "managed agent container pool is unavailable; refusing to run this task on the host"
|
|
d.Log.Error("task parked: managed worker unavailable", "task", task.ID,
|
|
"hint", "enable containers and make the archie-agent image available, then retry the task")
|
|
} else {
|
|
reason = "agent task transport is unavailable; refusing to run this task on the host"
|
|
d.Log.Error("task parked: agent task transport unavailable", "task", task.ID)
|
|
}
|
|
if err := d.Store.ParkTask(ctx, task.ID, workflow.StatusRunning, reason, taskstate.ParkTransient); err != nil {
|
|
d.Log.Warn("capability park transition failed", "task", task.ID, "err", err)
|
|
return
|
|
}
|
|
d.recordPark(ctx, task.ID, reason)
|
|
d.emit(events.Event{
|
|
Kind: events.KindParked, TaskID: task.ID, Attempt: task.Attempt,
|
|
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
|
|
Data: map[string]any{"park_class": taskstate.ParkTransient},
|
|
})
|
|
}
|
|
|
|
// parkRunningTask transitions a task from running to parked using a context
|
|
// detached from caller cancellation, with a bounded timeout.
|
|
//
|
|
// The park carries the caller's taskstate.ParkClass: every call site states
|
|
// what kind of intervention its park needs, and the store normalizes the
|
|
// class, so an unclassified caller cannot silently mislabel a park.
|
|
// When a task fails or is cancelled (e.g. via /stop or dashboard Stop), ctx is
|
|
// already cancelled. Writing terminal state using the cancelled ctx fails
|
|
// immediately in SQLite transactions and silently leaves the task row 'running'
|
|
// indefinitely. Like container teardown in Pool.Release, terminal state
|
|
// recording is cleanup on the way out of a task and must use a bounded context
|
|
// that survives cancellation while preserving values.
|
|
//
|
|
// The transition remains guarded from StatusRunning: if the worker already
|
|
// recorded a terminal state over storerpc, storecontract.ErrStaleTransition is returned
|
|
// and ignored. Unexpected store errors are logged as warnings.
|
|
func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason string, class taskstate.ParkClass) {
|
|
if d.Store == nil {
|
|
return
|
|
}
|
|
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
|
|
if err := d.Store.ParkTask(writeCtx, taskID, workflow.StatusRunning, reason, class); err != nil {
|
|
if !errors.Is(err, storecontract.ErrStaleTransition) {
|
|
d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err)
|
|
}
|
|
return
|
|
}
|
|
// Recorded only once the transition landed. An entry claiming the task
|
|
// parked would be a lie when another actor moved it out of running first
|
|
// -- the same race the parked event is withheld for.
|
|
d.recordPark(ctx, taskID, reason)
|
|
}
|
|
|
|
// recordPark writes one park reason into the task attempt's own log. Every
|
|
// place that parks a task calls it, including the two capability parks in
|
|
// process() that write their transition directly instead of going through
|
|
// parkRunningTask -- a park an operator cannot read the reason for is the same
|
|
// defect wherever it happens. Nothing daemon-side used to write to a task log
|
|
// at all: the container's system-log subscription was the only writer, so an
|
|
// attempt that parked before any container produced output left an empty log
|
|
// file and an unexplained park reason.
|
|
//
|
|
// A task with no open sink (a retry this instance never dispatched, or logging
|
|
// unwired) silently drops the entry, which is TaskRegistry.Write's own
|
|
// contract.
|
|
func (d *Daemon) recordPark(ctx context.Context, taskID int64, reason string) {
|
|
d.TaskLogs.Write(ctx, taskID, logging.Entry{
|
|
Time: time.Now(),
|
|
Level: slog.LevelError.String(),
|
|
Message: "task parked: " + reason,
|
|
Fields: map[string]any{"component": "daemon", "task": taskID},
|
|
})
|
|
}
|
|
|
|
// containerEnv returns the environment variables passed to agent containers.
|
|
// runViaAgent publishes a full-task handoff to archie-agent over the
|
|
// infrastructure-owned task subject and waits for its completion report. Every durable
|
|
// side effect (Store transitions, Forge calls, git push) happens inside
|
|
// archie-agent's workflow.Run, proxied back to this daemon over
|
|
// storerpc/forgerpc/worktreerpc -- those RPC servers are the sole place a
|
|
// terminal state gets written on success. runViaAgent only parks the task
|
|
// itself when nothing else could have: the request never reached (or was
|
|
// never answered by) an archie-agent, or archie-agent failed before its
|
|
// own workflow.Run got a chance to record an outcome.
|
|
func (d *Daemon) runViaAgent(ctx context.Context, task *workflow.Task, repo config.Repo, profile config.AgentProfile, harness *agentexec.HarnessSpec) {
|
|
grant, revoke, ok := d.publicationGrant(ctx, task)
|
|
if !ok {
|
|
return
|
|
}
|
|
defer revoke()
|
|
cfg := d.configFor(task)
|
|
taskCfg := cfg.ForTask()
|
|
d.captureAttemptConfig(ctx, task, taskCfg)
|
|
req := taskrun.Request{
|
|
Task: task,
|
|
Repo: repo,
|
|
Cfg: taskCfg,
|
|
Providers: agentexec.ProvidersFromConfig(cfg.Providers),
|
|
MCPServers: cfg.Tools.MCPServers,
|
|
WorktreeGrant: grant,
|
|
KindWorkflows: d.KindWorkflows,
|
|
LabelWorkflows: d.LabelWorkflows,
|
|
WorkflowDefinition: task.WorkflowDefinitionYAML,
|
|
Tools: profile.Tools,
|
|
Harness: harness,
|
|
}
|
|
data, err := json.Marshal(req)
|
|
if err != nil {
|
|
d.Log.Error("taskrun encode failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun encode failed: "+err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
|
|
reply, err := d.requestTaskRun(ctx, task.ID, data)
|
|
if cause := context.Cause(ctx); err != nil && (errors.Is(cause, errContainerExited) || errors.Is(cause, errTaskTimeLimit)) {
|
|
err = cause
|
|
}
|
|
if err != nil {
|
|
d.Log.Error("taskrun request failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
|
|
var resp taskrun.Response
|
|
if err := json.Unmarshal(reply, &resp); err != nil {
|
|
d.Log.Error("taskrun decode response failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun decode response failed: "+err.Error(), taskstate.ParkTransient)
|
|
return
|
|
}
|
|
if resp.Error != "" {
|
|
d.Log.Error("taskrun run failed", "task", task.ID, "err", resp.Error)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun run failed: "+resp.Error, taskstate.ParkTransient)
|
|
return
|
|
}
|
|
|
|
// Reached only past every failure return above, so this is always a
|
|
// fully decoded, error-free response -- a version is never recorded off
|
|
// a request that never reached an agent, or a response describing a
|
|
// parked/errored task.
|
|
if d.AgentStatus != nil && resp.AgentVersion != "" {
|
|
d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType)
|
|
}
|
|
|
|
d.Log.Info("taskrun complete", "task", task.ID, "status", resp.Status)
|
|
}
|
|
|
|
// pinTaskProfile pins the task's workflow definition and resolves the agent
|
|
// profile it names, parking the task when either fails. An unconfigured
|
|
// profile needs an operator: the fix is configuration, then a retry.
|
|
func (d *Daemon) pinTaskProfile(ctx context.Context, task *workflow.Task) (config.AgentProfile, bool) {
|
|
if err := d.pinWorkflowDefinition(ctx, task); err != nil {
|
|
d.Log.Error("pin workflow definition failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "pin workflow definition: "+err.Error(), taskstate.ParkTransient)
|
|
return config.AgentProfile{}, false
|
|
}
|
|
iface, err := workflowtask.ParseWorkflowInterface(task.WorkflowDefinitionYAML)
|
|
if err == nil {
|
|
// Every producer meets the declared interface here, including the
|
|
// ones that cannot read definitions when they enqueue (chat and
|
|
// dashboard spawns go through the Gateway, which never does).
|
|
err = iface.CheckInputs(task.Inputs)
|
|
}
|
|
if err == nil {
|
|
var profile config.AgentProfile
|
|
if profile, err = d.configFor(task).Containers.Profile(iface.Profile); err == nil {
|
|
if err = validateProfileMeetsNeeds(profile, iface.Needs()); err == nil {
|
|
return profile, true
|
|
}
|
|
}
|
|
}
|
|
d.Log.Error("agent profile unavailable", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "workflow "+task.Workflow+": "+err.Error(), taskstate.ParkNeedsHuman)
|
|
return config.AgentProfile{}, false
|
|
}
|
|
|
|
// validateProfileMeetsNeeds rejects a Kit profile whose harness cannot meet
|
|
// what the workflow declares it needs (docs/prds/external-agent-harness.md,
|
|
// "Contract"): a workflow needing captures but naming a profile whose Kit CLI
|
|
// serves none (no adapter, or one with no MCP config to register archie-agent
|
|
// mcp against -- e.g. Codex, Pi, OMP per agentexec.harnessAdapters). An image
|
|
// profile is never checked: the built-in agent loop always supports both
|
|
// captures and gate retries, so Needs constrains only the harness path.
|
|
func validateProfileMeetsNeeds(profile config.AgentProfile, needs workflowtask.WorkflowNeeds) error {
|
|
if !profile.IsKit() || !needs.Captures {
|
|
return nil
|
|
}
|
|
adapter, ok := agentexec.LookupHarnessAdapter(profile.Adapter)
|
|
if !ok || len(adapter.MCPConfig) == 0 {
|
|
return fmt.Errorf("needs.captures is set, but profile adapter %q serves no capture tools", profile.Adapter)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// publicationGrant issues the capability to publish the task's branch. A task
|
|
// with no repository has nothing to publish and gets none.
|
|
func (d *Daemon) publicationGrant(ctx context.Context, task *workflow.Task) (string, func(), bool) {
|
|
if !task.HasRepository() {
|
|
return "", func() {}, true
|
|
}
|
|
if d.WorktreeGrants == nil {
|
|
const reason = "worktree publication grants are unavailable"
|
|
d.Log.Error(reason, "task", task.ID)
|
|
d.parkRunningTask(ctx, task.ID, reason, taskstate.ParkTransient)
|
|
return "", nil, false
|
|
}
|
|
grant, revoke, err := d.WorktreeGrants.Issue(task)
|
|
if err != nil {
|
|
d.Log.Error("worktree publication grant failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "worktree publication grant failed: "+err.Error(), taskstate.ParkTransient)
|
|
return "", nil, false
|
|
}
|
|
return grant, revoke, true
|
|
}
|
|
|
|
// pinWorkflowDefinition puts the task's workflow definition pin in the state
|
|
// the worker's compile requires before it runs a dispatch: a stored definition
|
|
// that names the workflow the task row names
|
|
// (agentworker.CompilePinnedWorkflow).
|
|
//
|
|
// A pin that is present, hash-valid and id-consistent is reused unchanged, so
|
|
// the definition an operator had active when the task was pinned survives every
|
|
// later dispatch even after the active definition moved on. Anything else -- no
|
|
// pin, or a pin naming a workflow this task no longer runs -- is resolved again
|
|
// from the active collection for the workflow the task names.
|
|
//
|
|
// The re-pin is what makes the waiting_human -> approved requeue dispatchable:
|
|
// RequeueTask writes the new workflow column and leaves the previous run's
|
|
// definition pinned, so the requeued row carries the two half-state values the
|
|
// worker refuses to run. Resolving here, at the write, is where the row
|
|
// becomes valid; a consumer that repaired it would only be tolerating a state
|
|
// this producer is responsible for.
|
|
//
|
|
// An already-pinned definition is deliberately NOT refreshed to the operator's
|
|
// current active one: a run's definition is pinned once, so a mid-flight
|
|
// change to a definition never alters work already in progress.
|
|
func (d *Daemon) pinWorkflowDefinition(ctx context.Context, task *workflow.Task) error {
|
|
pinned, ok, err := pinnedDefinitionID(task)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if ok && pinned == task.Workflow {
|
|
return nil
|
|
}
|
|
d.runActionPlaybooks(ctx, task)
|
|
if d.WorkflowDefinitions == nil {
|
|
return d.pinWorkflowFromCollection(ctx, task, workflow.ShippedDefinitions(), 0)
|
|
}
|
|
collection, version, err := d.WorkflowDefinitions.WorkflowDefinitions(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return d.pinWorkflowFromCollection(ctx, task, collection, version)
|
|
}
|
|
|
|
// pinnedDefinitionID reports the workflow a task's stored definition pin names,
|
|
// with ok false for a task carrying no pin. It enforces what this package owns
|
|
// about a stored pin: the YAML hashes to the digest recorded beside it, and it
|
|
// declares an id at all. A pin failing either is a row an operator has to
|
|
// resolve, not one to silently overwrite.
|
|
func pinnedDefinitionID(task *workflow.Task) (string, bool, error) {
|
|
if task.WorkflowDefinitionYAML == "" {
|
|
return "", false, nil
|
|
}
|
|
if workflow.DigestDefinition(task.WorkflowDefinitionYAML) != task.WorkflowDefinitionDigest {
|
|
return "", false, fmt.Errorf("stored workflow definition digest mismatch")
|
|
}
|
|
id, err := workflow.DefinitionID(task.WorkflowDefinitionYAML)
|
|
if err != nil {
|
|
return "", false, fmt.Errorf("stored workflow definition: %w", err)
|
|
}
|
|
return id, true, nil
|
|
}
|
|
|
|
// runActionPlaybooks runs the action playbooks matching a task that is about
|
|
// to be pinned, which happens once per task (docs/prds/action-playbook-run.md).
|
|
// A run never affects routing and its failure never fails the task. A task
|
|
// that already names a workflow is the approval requeue and is skipped, as
|
|
// it is for workflow playbooks.
|
|
func (d *Daemon) runActionPlaybooks(ctx context.Context, task *workflow.Task) {
|
|
if d.Playbooks == nil || d.PlaybookLedger == nil || task.Workflow != "" {
|
|
return
|
|
}
|
|
if err := d.Playbooks.Run(ctx, d.PlaybookLedger, d.Log, playbookInput(task)); err != nil {
|
|
d.Log.Warn("action playbook run failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
|
|
// resolveWorkflowID picks the definition id to pin. A playbook whose trigger
|
|
// matches the task decides, ahead of the kind/label bindings; anything else
|
|
// falls through to workflow.ResolveWorkflowID unchanged.
|
|
//
|
|
// A task that already names a workflow is skipped entirely: that assignment is
|
|
// the waiting_human -> approved requeue handoff, a decision already made about
|
|
// this specific task, and a trigger that still matches its labels must not
|
|
// overturn it.
|
|
func (d *Daemon) resolveWorkflowID(task *workflow.Task, available map[string]struct{}) (string, error) {
|
|
if d.Playbooks == nil || task.Workflow != "" {
|
|
return workflow.ResolveWorkflowID(task, available, d.KindWorkflows, d.LabelWorkflows)
|
|
}
|
|
decision, matched := d.Playbooks.Dispatch(playbookInput(task))
|
|
if !matched {
|
|
return workflow.ResolveWorkflowID(task, available, d.KindWorkflows, d.LabelWorkflows)
|
|
}
|
|
if _, defined := available[decision.Workflow]; !defined {
|
|
// Reported, not silently downgraded to the binding's choice: the
|
|
// operator bound this trigger to a named workflow, and running a
|
|
// different one under their rule is worse than parking the task.
|
|
return "", fmt.Errorf("playbook %q binds this trigger to workflow %q, which is not defined", decision.PlaybookID, decision.Workflow)
|
|
}
|
|
d.Log.Info("playbook selected workflow",
|
|
"task", task.ID, "playbook", decision.PlaybookID,
|
|
"playbook_version", decision.Version, "workflow", decision.Workflow)
|
|
return decision.Workflow, nil
|
|
}
|
|
|
|
// playbookInput builds the coordinator's dispatch input from a task row. The
|
|
// event surface is what the intake point actually knows about the originating
|
|
// issue -- the fields a `when` condition can read; it is deliberately not
|
|
// padded with values the daemon would have to invent.
|
|
func playbookInput(task *workflow.Task) playbook.DispatchInput {
|
|
labels := workintake.SplitLabels(task.Labels)
|
|
kind := string(workintake.KindForLabels(labels))
|
|
return playbook.DispatchInput{
|
|
Labels: labels,
|
|
Kind: kind,
|
|
TaskID: workintake.TaskEnvelope{Org: task.Org, Identity: task.Identity, Owner: task.Owner, Repo: task.Repo, Number: task.IssueNumber}.IdempotencyKey(),
|
|
Event: map[string]any{
|
|
"kind": kind,
|
|
"labels": labels,
|
|
"owner": task.Owner,
|
|
"repo": task.Repo,
|
|
"number": task.IssueNumber,
|
|
"title": task.Title,
|
|
"body": task.Body,
|
|
"source": task.Source,
|
|
},
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) pinWorkflowFromCollection(ctx context.Context, task *workflow.Task, collection workflow.WorkflowDefinitionCollection, version int64) error {
|
|
available := make(map[string]struct{}, len(collection.Definitions))
|
|
for _, definition := range collection.Definitions {
|
|
available[definition.ID] = struct{}{}
|
|
}
|
|
id, err := d.resolveWorkflowID(task, available)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
definition, ok := collection.DefinitionByID(id)
|
|
if !ok {
|
|
return fmt.Errorf("workflow definition %q disappeared", id)
|
|
}
|
|
task.Workflow = id
|
|
task.WorkflowDefinitionVersion = version
|
|
task.WorkflowDefinitionYAML = definition.YAML
|
|
task.WorkflowDefinitionDigest = workflow.DigestDefinition(definition.YAML)
|
|
if err := d.Store.Update(ctx, task); err != nil {
|
|
return fmt.Errorf("persist workflow definition pin: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
const (
|
|
// defaultTaskRunReadyTimeout is how long requestTaskRun retries a
|
|
// nats.ErrNoResponders before giving up, when Daemon.TaskRunReadyTimeout
|
|
// is unset.
|
|
defaultTaskRunReadyTimeout = 20 * time.Second
|
|
// defaultTaskRunRetryBackoff is the delay between retry attempts, when
|
|
// Daemon.TaskRunRetryBackoff is unset.
|
|
defaultTaskRunRetryBackoff = 250 * time.Millisecond
|
|
)
|
|
|
|
func (d *Daemon) taskRunReadyTimeout() time.Duration {
|
|
if d.TaskRunReadyTimeout > 0 {
|
|
return d.TaskRunReadyTimeout
|
|
}
|
|
return defaultTaskRunReadyTimeout
|
|
}
|
|
|
|
func (d *Daemon) taskRunRetryBackoff() time.Duration {
|
|
if d.TaskRunRetryBackoff > 0 {
|
|
return d.TaskRunRetryBackoff
|
|
}
|
|
return defaultTaskRunRetryBackoff
|
|
}
|
|
|
|
// requestTaskRun publishes the taskrun request and retries while no
|
|
// archie-agent has subscribed yet (nats.ErrNoResponders): the container
|
|
// pool's Acquire returns as soon as Docker has issued the start syscall,
|
|
// not once the spawned container has connected to core NATS and subscribed
|
|
// to this per-task subject --
|
|
// a gap of hundreds of milliseconds to a few seconds that would otherwise
|
|
// fail the very first request deterministically, every time, on every
|
|
// task. Any error other than ErrNoResponders (encode failures, a context
|
|
// that's already done, etc.) is returned immediately without retrying,
|
|
// since those don't mean "not ready yet".
|
|
func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) {
|
|
subject := agentnats.SubjectForTask(taskID)
|
|
deadline := time.Now().Add(d.taskRunReadyTimeout())
|
|
backoff := d.taskRunRetryBackoff()
|
|
for {
|
|
reply, err := d.Tasks.Request(ctx, subject, data)
|
|
if err == nil {
|
|
return reply, nil
|
|
}
|
|
if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) {
|
|
return nil, err
|
|
}
|
|
wait := min(backoff, time.Until(deadline))
|
|
if wait <= 0 {
|
|
return nil, err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-time.After(wait):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) containerEnv(task *workflow.Task, stateStoreToken string) []string {
|
|
var env []string
|
|
// The endpoint the daemon's own client connected with at startup, not
|
|
// the live config: a reloaded [nats] section must not point new
|
|
// containers at a server the daemon is not publishing on.
|
|
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
|
if token := d.ConnectedNATS.Token; token != "" {
|
|
env = append(env, "NATS_TOKEN="+token)
|
|
}
|
|
// The State Store gRPC target follows the same seam as NATS_URL/
|
|
// NATS_TOKEN above (docs/prds/state-store-contract.md §6). It is always
|
|
// the configured [services.state].target after the in-process serving
|
|
// path is deleted. stateStoreToken is this task's own scoped credential
|
|
// (see stateStoreGrantToken) -- never d.ConnectedStateStore.Token, which
|
|
// is the daemon's administrative credential and must never reach a
|
|
// container.
|
|
if d.ConnectedStateStore.URL != "" {
|
|
env = append(env, "STATE_STORE_URL="+d.ConnectedStateStore.URL)
|
|
if stateStoreToken != "" {
|
|
env = append(env, "STATE_STORE_TOKEN="+stateStoreToken)
|
|
}
|
|
}
|
|
// The agent process runs as root inside the container (no USER in the
|
|
// Dockerfile, no userns-remap), so a commit it writes to the bind-mounted
|
|
// worktree lands owned by UID 0 on the host. The daemon then reads those
|
|
// same loose objects to push, running as its own non-root host user --
|
|
// archie-core#520's "permission denied" on .git/objects. Passing the
|
|
// daemon's own UID/GID lets the agent chown the worktree back to it
|
|
// before pushing.
|
|
env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()))
|
|
env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid()))
|
|
for _, p := range d.configFor(task).Providers {
|
|
if p.APIKeyEnv != "" {
|
|
if v := os.Getenv(p.APIKeyEnv); v != "" {
|
|
env = append(env, p.APIKeyEnv+"="+v)
|
|
}
|
|
}
|
|
}
|
|
return env
|
|
}
|
|
|
|
// stateStoreGrantToken issues this task's scoped State Store credential, or
|
|
// ("", no-op revoke, nil) when no State Store is configured. It fails closed
|
|
// -- rather than falling back to the daemon's administrative token -- when a
|
|
// State Store is configured but no StateStoreGrantIssuer is wired, which is
|
|
// a composition bug, not a runtime condition to paper over.
|
|
func (d *Daemon) stateStoreGrantToken(task *workflow.Task) (string, func(), error) {
|
|
if d.ConnectedStateStore.URL == "" {
|
|
return "", func() {}, nil
|
|
}
|
|
if d.StateStoreGrants == nil {
|
|
return "", nil, fmt.Errorf("state store is configured but no task-scoped grant issuer is wired")
|
|
}
|
|
return d.StateStoreGrants.Issue(task)
|
|
}
|
|
|
|
func (d *Daemon) configFor(task *workflow.Task) config.Config {
|
|
if id := d.identityFor(task); id != nil {
|
|
return configForIdentity(d.Cfg.Get(), id.Cfg)
|
|
}
|
|
return d.Cfg.Get()
|
|
}
|
|
|
|
// captureAttemptConfig persists the effective task configuration this attempt
|
|
// runs under, as a durable event on the attempt's own key.
|
|
//
|
|
// It is written where the configuration is materialised for the dispatch, so
|
|
// the document is exactly what the run received. The published configuration
|
|
// snapshot cannot answer this: it is the CURRENT configuration, replaced on
|
|
// every publish, so a run that finished last week has no record anywhere else.
|
|
//
|
|
// Fail-open by construction: a configuration that could not be recorded is
|
|
// worth a log line and nothing more. Parking a task over a reporting failure
|
|
// would trade the run for the receipt.
|
|
func (d *Daemon) captureAttemptConfig(ctx context.Context, task *workflow.Task, cfg config.TaskConfig) {
|
|
if d.Store == nil {
|
|
return
|
|
}
|
|
document, err := json.Marshal(cfg)
|
|
if err != nil {
|
|
d.Log.Warn("attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
|
|
return
|
|
}
|
|
event := events.Event{
|
|
At: time.Now().UTC(),
|
|
Kind: events.KindConfigCaptured,
|
|
TaskID: task.ID,
|
|
Repo: task.Owner + "/" + task.Repo,
|
|
Issue: task.IssueNumber,
|
|
Workflow: task.Workflow,
|
|
Attempt: task.Attempt,
|
|
Data: map[string]any{
|
|
"schema": events.ConfigCapturedSchema,
|
|
// Raw so the document is embedded once, as it was encoded, rather
|
|
// than as a string a reader would have to decode again.
|
|
"document": json.RawMessage(document),
|
|
},
|
|
}
|
|
id, err := d.Store.InsertEvent(ctx, event)
|
|
if err != nil {
|
|
d.Log.Warn("attempt configuration not captured", "task", task.ID, "attempt", task.Attempt, "err", err)
|
|
return
|
|
}
|
|
// The assigned ID tells the event sink to broadcast without inserting a
|
|
// duplicate row.
|
|
event.ID = id
|
|
d.emit(event)
|
|
}
|
|
|
|
func configForIdentity(root config.Config, identity config.IdentityConfig) config.Config {
|
|
root.BotUser = identity.BotUser
|
|
root.BotEmail = identity.BotEmail
|
|
// Only an identity that set the cap overrides the shared one. Assigning
|
|
// unconditionally made an identity that omits the key inherit a nil cap,
|
|
// which reads as "no cap" and silently disabled the safety rail.
|
|
if identity.DiffCapLines != nil {
|
|
root.DiffCapLines = identity.DiffCapLines
|
|
}
|
|
if identity.Org != "" {
|
|
root.Org = identity.Org
|
|
}
|
|
if identity.GrantedCredentials != nil {
|
|
root.GrantedCredentials = *identity.GrantedCredentials
|
|
}
|
|
root.Forge = identity.Forge
|
|
root.Dispatch = identity.Dispatch
|
|
root.Models = identity.Models
|
|
root.Providers = identity.Providers
|
|
root.Budgets = identity.Budgets
|
|
root.Notify = identity.Notify
|
|
root.Repos = identity.Repos
|
|
return root
|
|
}
|
|
|
|
// identityFor resolves the IdentityRunner that owns task, or nil for
|
|
// single-identity deployments and forge-sourced tasks recorded
|
|
// before multi-identity routing existed (task.Identity == ""). Callers
|
|
// must fall back to the root d.Forge/d.Trees/d.Cfg when this returns nil.
|
|
func (d *Daemon) identityFor(task *workflow.Task) *IdentityRunner {
|
|
if task == nil || task.Identity == "" {
|
|
return nil
|
|
}
|
|
for _, id := range d.Identities {
|
|
if id.Name == task.Identity || string(id.ID) == task.Identity {
|
|
return id
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// identityMayAct reports whether the identity a task carries may be acted on.
|
|
// A denial also carries the park class the task would need: a name that no
|
|
// longer resolves is fixable by re-adding the identity (transient), while a
|
|
// retired or deactivated identity is a deliberate operator state (terminal).
|
|
// An empty identity is the single-identity root and always may act. A
|
|
// non-empty identity must resolve to a configured runner and pass the
|
|
// lifecycle check; otherwise the returned reason says why it may not. This is
|
|
// the single task-side fail-closed point, mirroring identity.Resolve's
|
|
// credential-side behaviour: a task naming an identity archie no longer knows
|
|
// -- renamed, retired, or deleted in the control plane -- must park, never
|
|
// fall back to the root forge's credential.
|
|
func (d *Daemon) identityMayAct(ctx context.Context, name string) (bool, string, taskstate.ParkClass) {
|
|
if name == "" {
|
|
return true, "", ""
|
|
}
|
|
runner := d.identityFor(&workflow.Task{Identity: name})
|
|
if runner == nil {
|
|
return false, "identity " + name + " no longer resolves", taskstate.ParkTransient
|
|
}
|
|
if !d.identityActive(ctx, runner.ID) {
|
|
return false, "identity " + runner.Name + " may not act", taskstate.ParkTerminal
|
|
}
|
|
return true, "", ""
|
|
}
|
|
|
|
// parkIfIdentityUnresolvable parks task when its non-empty identity cannot be
|
|
// acted on, reporting whether it did. An empty identity is the root
|
|
// single-identity path and never parks here.
|
|
func (d *Daemon) parkIfIdentityUnresolvable(ctx context.Context, task *workflow.Task) bool {
|
|
ok, reason, class := d.identityMayAct(ctx, task.Identity)
|
|
if ok {
|
|
return false
|
|
}
|
|
d.parkRunningTask(ctx, task.ID, reason, class)
|
|
return true
|
|
}
|
|
|
|
func (d *Daemon) identityActive(ctx context.Context, id identity.IdentityID) bool {
|
|
if d.IdentityRepository == nil {
|
|
return true
|
|
}
|
|
if id == "" {
|
|
d.Log.Error("identity lifecycle check failed", "err", "identity ID is empty")
|
|
return false
|
|
}
|
|
value, err := d.IdentityRepository.Get(ctx, id)
|
|
if err != nil {
|
|
d.Log.Error("identity lifecycle check failed", "identity", id, "err", err)
|
|
return false
|
|
}
|
|
return value.Lifecycle == identity.LifecycleActive
|
|
}
|
|
|
|
// forgeFor returns the forge client that owns task: the identity's own
|
|
// client when task.Identity names a configured identity, else the root
|
|
// d.Forge. This is the safety boundary that keeps one identity's forge
|
|
// token from being used against another identity's repos.
|
|
func (d *Daemon) forgeFor(task *workflow.Task) forge.Forge {
|
|
if id := d.identityFor(task); id != nil {
|
|
return id.Forge
|
|
}
|
|
return d.Forge
|
|
}
|
|
|
|
// treesFor returns the worktree manager that owns task, mirroring forgeFor.
|
|
func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager {
|
|
if id := d.identityFor(task); id != nil {
|
|
return id.Trees
|
|
}
|
|
return d.Trees
|
|
}
|
|
|
|
// repoFor resolves task's repo config from the owning identity's repo
|
|
// list when task.Identity is set, else the root Cfg.Repos.
|
|
func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) {
|
|
repos := d.Cfg.Get().Repos
|
|
if id := d.identityFor(t); id != nil {
|
|
repos = id.Repos
|
|
}
|
|
for _, r := range repos {
|
|
if r.Owner == t.Owner && r.Name == t.Repo {
|
|
return r, true
|
|
}
|
|
}
|
|
return config.Repo{}, false
|
|
}
|
|
|
|
// allowConcurrentForTask reports whether the task's owning repo has opted
|
|
// into concurrent task dispatch (config.Repo.AllowConcurrent). The repo is
|
|
// resolved through repoFor, which selects the identity-scoped repo list when
|
|
// task.Identity names a configured identity -- so multi-identity concurrency
|
|
// policy comes from the identity's own config, not the root daemon's.
|
|
// Unknown repos default to the safe, serialized behavior.
|
|
func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
|
|
if !task.HasRepository() {
|
|
return true
|
|
}
|
|
repo, ok := d.repoFor(task)
|
|
return ok && repo.AllowConcurrent
|
|
}
|
|
|
|
// LastPollAt reports when the daemon last began a poll pass, or the zero
|
|
// time when no pass has started yet.
|
|
func (d *Daemon) LastPollAt() time.Time {
|
|
ns := d.lastPollAt.Load()
|
|
if ns == 0 {
|
|
return time.Time{}
|
|
}
|
|
return time.Unix(0, ns).UTC()
|
|
}
|
|
|
|
// markPoll stamps the start of a poll pass. It is called from both poll
|
|
// paths: Run's single-identity loop (via poll) and runIdentities' per-identity
|
|
// goroutines (via pollForIdentity).
|
|
//
|
|
// The stamp is the pass's START, not its completion: a pass that hangs on an
|
|
// unreachable forge leaves the stamp stale, which is exactly the signal an
|
|
// operator needs. Stamping the completion instead would leave the last
|
|
// successful pass's time in place, so a wedged poller would look healthy.
|
|
func (d *Daemon) markPoll() {
|
|
d.lastPollAt.Store(time.Now().UnixNano())
|
|
}
|
|
|
|
// errContainerExited is the cancellation cause when a task's agent container
|
|
// stops before answering its taskrun request.
|
|
var errContainerExited = errors.New("agent container exited before the task finished")
|
|
|
|
// withContainerExit derives a context cancelled when exited closes. A dead
|
|
// container never answers its taskrun request, so without this the request
|
|
// blocks forever and the task row stays running.
|
|
func withContainerExit(ctx context.Context, exited <-chan struct{}) (context.Context, func()) {
|
|
runCtx, cancel := context.WithCancelCause(ctx)
|
|
go func() {
|
|
select {
|
|
case <-exited:
|
|
cancel(errContainerExited)
|
|
case <-runCtx.Done():
|
|
}
|
|
}()
|
|
return runCtx, func() { cancel(nil) }
|
|
}
|
|
|
|
// errTaskTimeLimit is the cancellation cause when a task outruns its overall
|
|
// time limit (the execution settings' max task runtime).
|
|
var errTaskTimeLimit = errors.New("task exceeded its time limit")
|
|
|
|
// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies
|
|
// no bound.
|
|
func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) {
|
|
if limit <= 0 {
|
|
return ctx, func() {}
|
|
}
|
|
return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit))
|
|
}
|
|
|
|
// RemoveWorktree deletes a task's clone from the trees of the identity that
|
|
// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold
|
|
// uncaptured work, and the operator's archive is the decision to discard it.
|
|
func (d *Daemon) RemoveWorktree(owner, repo, identity string, issue int) error {
|
|
trees := d.treesFor(&workflow.Task{Owner: owner, Repo: repo, Identity: identity})
|
|
if trees == nil {
|
|
return nil
|
|
}
|
|
return trees.Cleanup(owner, repo, issue)
|
|
}
|