Root-cause fix for the correctness gap named in archie-core-t2db.8: resolveBindingRepo refused to dispatch with zero or multiple configured repos, meaning a multi-repo deployment could never use bindings at all. - binding.Binding gains optional Owner/Repo fields; Validate rejects setting only one (a pin is a complete pair or not a pin at all). - Persisted as new owner/repo columns (NOT NULL DEFAULT '', additive -- no migration needed since bindings is a CREATE TABLE IF NOT EXISTS with no existing deployment yet). - resolveBindingRepo now takes the binding: a pinned binding dispatches to its own owner/repo unconditionally; an unpinned binding keeps today's single-configured-repo fallback (0 or 2+ repos still refuses, unchanged -- verified by a dedicated regression test, not just assumed preserved). - Wired through the API request/response shape (create/update accept and return owner/repo); no repo-existence validation against config added -- smallest workable change, not the named bug. Red-green throughout: 8 new tests across domain/store/daemon/webui layers, each proven to fail before the corresponding implementation step. Zero regressions in existing binding/dispatch test suites. Dashboard UI to expose this (the other half of t2db.8) is unstarted -- this closes the backend correctness gap only.
1486 lines
54 KiB
Go
1486 lines
54 KiB
Go
// Package daemon is archied's resident loop: poll GitHub for labelled
|
|
// issues, enqueue them, and process tasks through their routed workflows.
|
|
// State lives in the store; the daemon is restartable at any point.
|
|
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/container"
|
|
"github.com/samcharles93/archie-core/internal/domain/binding"
|
|
"github.com/samcharles93/archie-core/internal/domain/curator"
|
|
"github.com/samcharles93/archie-core/internal/domain/mapping"
|
|
"github.com/samcharles93/archie-core/internal/domain/workintake"
|
|
"github.com/samcharles93/archie-core/internal/eventbus"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/forge"
|
|
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
|
"github.com/samcharles93/archie-core/internal/logging"
|
|
"github.com/samcharles93/archie-core/internal/plugin"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
"github.com/samcharles93/archie-core/internal/store"
|
|
"github.com/samcharles93/archie-core/internal/taskrun"
|
|
"github.com/samcharles93/archie-core/internal/tools"
|
|
"github.com/samcharles93/archie-core/internal/worktree"
|
|
)
|
|
|
|
// TaskBus is the messaging the daemon needs: announce discovered work,
|
|
// collect it back, and ask a worker to run a task.
|
|
//
|
|
// Declared here rather than taking eventbus.Bus because a domain defines the
|
|
// smallest interface required to do its work -- the daemon never subscribes
|
|
// or creates reply inboxes.
|
|
type TaskBus interface {
|
|
PublishUnique(ctx context.Context, subject, idempotencyKey string, payload []byte) error
|
|
Fetch(ctx context.Context) (eventbus.Message, error)
|
|
Request(ctx context.Context, subject string, payload []byte) ([]byte, error)
|
|
}
|
|
|
|
// WorktreeGrantIssuer gives one container dispatch the narrow authority to
|
|
// publish its already-prepared branch. The daemon owns grant lifetime; the
|
|
// transport implementation owns token mechanics.
|
|
type WorktreeGrantIssuer interface {
|
|
Issue(task *store.Task) (token string, revoke func(), err error)
|
|
}
|
|
|
|
// NATSEndpoint is the broker address and credential archied connected with at
|
|
// startup. It is runtime state rather than configuration: embedded mode
|
|
// generates both values, and external secret references are resolved once.
|
|
type NATSEndpoint struct {
|
|
URL string
|
|
Token string
|
|
}
|
|
|
|
type Daemon struct {
|
|
// Cfg publishes the running configuration. Read through Cfg() so a
|
|
// reload swaps the published snapshot atomically. See config.Holder.
|
|
Cfg *config.Holder
|
|
// ConnectedNATS is the endpoint and credential the daemon's own client
|
|
// connected with at startup. Container env is built from this, not from
|
|
// live config or environment, so reload cannot point new containers at a
|
|
// broker the daemon is not publishing on. The zero value is invalid in
|
|
// production composition and remains useful only to fail closed in tests.
|
|
ConnectedNATS NATSEndpoint
|
|
Store store.TaskStore
|
|
// Mappings persists payload field mappings (docs/prds/payload-field-mapping.md).
|
|
// Used by the binding dispatch loop to resolve capture bodies against
|
|
// the mapping a binding names. Optional: nil disables the binding
|
|
// dispatch loop (legacy behaviour).
|
|
Mappings store.MappingStore
|
|
// Bindings persists playbook bindings (docs/prds/playbook-binding.md).
|
|
// Optional: nil disables the binding dispatch loop (legacy behaviour).
|
|
Bindings store.BindingStore
|
|
// BindingDispatcher is the dispatch-time helper surface for bindings:
|
|
// list undispatched captures, look up armed bindings by source for the
|
|
// matcher, and write the at-most-once dedup ledger row. Split from
|
|
// BindingStore to keep the CRUD interface under the interfacebloat
|
|
// limit. Optional: nil disables the dispatch loop.
|
|
BindingDispatcher store.BindingDispatcher
|
|
// BindingTaskCreator enqueues a task triggered by a binding. Split
|
|
// off TaskLifecycle so the lifecycle surface stays narrow and the
|
|
// dispatch loop does not acquire the full task-creation contract.
|
|
// Optional: nil disables the dispatch loop.
|
|
BindingTaskCreator store.BindingTaskCreator
|
|
Forge forge.Forge
|
|
Trees *worktree.Manager
|
|
Bus *events.Bus
|
|
Log *slog.Logger
|
|
// Tasks is the optional message bus for task distribution. Nil means no
|
|
// bus is configured; the existing SQLite ClaimNext flow is used.
|
|
Tasks TaskBus
|
|
WorktreeGrants WorktreeGrantIssuer
|
|
// TaskRunReadyTimeout bounds how long runViaAgent retries an initial
|
|
// taskrun request that fails with nats.ErrNoResponders, giving a
|
|
// freshly spawned archie-agent container time to connect to NATS, set
|
|
// up its task-scoped core-NATS subscription, and become reachable before the daemon
|
|
// gives up and parks the task. ContainerPool.Acquire returns as soon
|
|
// as Docker has issued the start syscall, not once that setup
|
|
// finishes, so without this bound the very first request after a
|
|
// container spawn fails deterministically. Zero uses
|
|
// defaultTaskRunReadyTimeout.
|
|
TaskRunReadyTimeout time.Duration
|
|
// TaskRunRetryBackoff is the delay between retry attempts within
|
|
// TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff.
|
|
TaskRunRetryBackoff time.Duration
|
|
// AgentStatus records the version/install-type the most recently
|
|
// completed archie-agent task response reported about itself, for
|
|
// releaseupdate.Report.Verify to check an update claim against. Nil
|
|
// disables observation -- runViaAgent skips the Observe call rather
|
|
// than reporting the agent as unverifiable via a zero value, which
|
|
// would read as a checked "unknown" rather than "never wired".
|
|
AgentStatus *AgentStatus
|
|
|
|
// ContainerPool manages Docker container lifecycle. Nil means autonomous
|
|
// execution is unavailable and process parks tasks; every runnable task gets
|
|
// a fresh container.
|
|
ContainerPool *container.Pool
|
|
// Storage is the pluggable storage backend for container mounts.
|
|
// A runnable task requires it; acquireTaskContainer parks on nil.
|
|
Storage storage.Backend
|
|
// CapabilityHost owns validated plugin manifests and cross-family
|
|
// lifecycle. Typed capability registries remain in their domain packages;
|
|
// the host never exposes daemon internals or an untyped service locator.
|
|
CapabilityHost *plugin.Host
|
|
// Curators is the curator engine family registry (epic archie-core-yp9).
|
|
// The runtime loop (archie-core-89x) and reference curators (i7i, gs8)
|
|
// are driven through it; nil when the family is not composed.
|
|
Curators *curator.Registry
|
|
// Identities holds per-identity runner state for multi-identity mode.
|
|
// When non-empty, Run() starts one goroutine per identity instead of
|
|
// using the single-identity Forge/Trees/Cfg.Repos path.
|
|
Identities []*IdentityRunner
|
|
|
|
// Guardrails is the tool-call guardrail engine, wired by the composition
|
|
// root. When non-nil, tool successes and failures are recorded and
|
|
// warnings/hard-stops are issued per the configured thresholds. Nil
|
|
// means guardrails are disabled (backward compatible).
|
|
Guardrails *tools.GuardrailEngine
|
|
|
|
// ToolRegistry is the central tool registry, wired by the composition
|
|
// root. MCP-discovered tools and built-in tools are registered here
|
|
// and passed as CaptureTools in agent requests. Nil means no dynamic
|
|
// tool discovery (backward compatible).
|
|
ToolRegistry *tools.Registry
|
|
|
|
// TaskLogs persists each task's own log output (including a sandboxed
|
|
// container's, which otherwise disappears at AutoRemove) and mirrors it
|
|
// live onto the dashboard feed, wired by the composition root. Nil
|
|
// disables task logging (backward compatible); every TaskRegistry
|
|
// method is nil-receiver-safe, so process() calls it unconditionally.
|
|
TaskLogs *logging.TaskRegistry
|
|
|
|
// running holds a cancel function for every task currently executing,
|
|
// so /stop can reach work already in flight. Its zero value is ready
|
|
// to use.
|
|
running runningTasks
|
|
}
|
|
|
|
// IdentityRunner bundles identity-specific state for a single agent
|
|
// identity running within a multi-identity daemon. Each identity gets its
|
|
// own forge client, worktree manager, repo list, and config -- but shares
|
|
// the store, NATS connection, container pool, and event bus with siblings.
|
|
type IdentityRunner struct {
|
|
Name string
|
|
Forge forge.Forge
|
|
Trees *worktree.Manager
|
|
Repos []config.Repo
|
|
// Cfg is the identity-scoped config subset (forge, models, dispatch,
|
|
// budgets, etc.).
|
|
Cfg config.IdentityConfig
|
|
Log *slog.Logger
|
|
}
|
|
|
|
// NewIdentityRunner constructs an IdentityRunner from an identity config
|
|
// and a pre-built forge client. The caller owns forge and trees lifecycle;
|
|
// IdentityRunner just holds references.
|
|
func NewIdentityRunner(ctx context.Context, idCfg config.IdentityConfig, fg forge.Forge, trees *worktree.Manager, log *slog.Logger) (*IdentityRunner, error) {
|
|
if idCfg.Name == "" {
|
|
return nil, fmt.Errorf("identity name is required")
|
|
}
|
|
return &IdentityRunner{
|
|
Name: idCfg.Name,
|
|
Forge: fg,
|
|
Trees: trees,
|
|
Repos: idCfg.Repos,
|
|
Cfg: idCfg,
|
|
Log: log.With("identity", idCfg.Name),
|
|
}, nil
|
|
}
|
|
|
|
// emit publishes an observability event; safe on a nil bus.
|
|
func (d *Daemon) emit(e events.Event) {
|
|
if d.Bus != nil {
|
|
d.Bus.Publish(e)
|
|
}
|
|
}
|
|
|
|
// Startup runs crash recovery and access verification once.
|
|
func (d *Daemon) Startup(ctx context.Context) error {
|
|
d.cleanupExpiredStorage(ctx)
|
|
n, err := d.Store.RecoverStale(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
d.Log.Info("re-queued tasks left running by a previous daemon", "count", n)
|
|
}
|
|
if err := d.Forge.AcceptInvitations(ctx); err != nil {
|
|
d.Log.Warn("invitation sweep failed", "err", err)
|
|
}
|
|
for _, r := range d.Cfg.Get().Repos {
|
|
if err := d.Forge.VerifyPush(ctx, r.Owner, r.Name); err != nil {
|
|
d.Log.Warn("repo not pushable -- tasks from it will fail", "repo", r.FullName(), "err", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Run polls until the context ends. When Identities is non-empty, each
|
|
// identity gets its own goroutine with its own forge client, repo list,
|
|
// and poll interval -- failure isolation between identities (one identity's
|
|
// forge outage doesn't block another's poll cycle).
|
|
func (d *Daemon) Run(ctx context.Context) error {
|
|
if len(d.Identities) > 0 {
|
|
return d.runIdentities(ctx)
|
|
}
|
|
interval := d.Cfg.Get().PollInterval.Std()
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
d.Cycle(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
// PollInterval may have been reloaded; reset the ticker so a
|
|
// change takes effect on the next cycle.
|
|
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
|
|
interval = iv
|
|
ticker.Reset(iv)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// runIdentities starts one goroutine per identity plus one shared
|
|
// maintenance-and-drain loop.
|
|
//
|
|
// Polling is identity-scoped: each identity goroutine polls only its own
|
|
// repos via its own forge client and enqueues under its own name. It never
|
|
// drains or reconciles. Maintenance and draining are store-wide, not
|
|
// identity-scoped, so they run once in a single shared loop -- that keeps
|
|
// the global max_concurrency cap and the per-repo serialization intact
|
|
// across identities (N per-identity dispatchers would multiply the cap and
|
|
// split the same-repo lock), and it ensures reconcilePRs and storage
|
|
// cleanup actually run in multi-identity mode.
|
|
func (d *Daemon) runIdentities(ctx context.Context) error {
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
// One poll loop per identity. Polling is the only identity-scoped
|
|
// work; it never executes tasks.
|
|
for _, id := range d.Identities {
|
|
wg.Add(1)
|
|
go func(id *IdentityRunner) {
|
|
defer wg.Done()
|
|
interval := d.Cfg.Get().PollInterval.Std()
|
|
if id.Cfg.PollInterval > 0 {
|
|
interval = id.Cfg.PollInterval.Std()
|
|
}
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
d.pollForIdentity(ctx, id)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
// The identity's own override is frozen (identity config is
|
|
// startup-built); the root fallback may have been reloaded.
|
|
iv := d.Cfg.Get().PollInterval.Std()
|
|
if id.Cfg.PollInterval > 0 {
|
|
iv = id.Cfg.PollInterval.Std()
|
|
}
|
|
if iv != interval {
|
|
interval = iv
|
|
ticker.Reset(iv)
|
|
}
|
|
}
|
|
}
|
|
}(id)
|
|
}
|
|
|
|
// One shared maintenance-and-drain loop.
|
|
wg.Go(func() {
|
|
interval := d.Cfg.Get().PollInterval.Std()
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
d.maintainAndDrain(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
// PollInterval may have been reloaded; reset the ticker so a
|
|
// change takes effect on the next cycle.
|
|
if iv := d.Cfg.Get().PollInterval.Std(); iv != interval {
|
|
interval = iv
|
|
ticker.Reset(iv)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
wg.Wait()
|
|
return ctx.Err()
|
|
}
|
|
|
|
// pollForIdentity polls one identity's repos via its own forge client,
|
|
// enqueuing any discovered issues under that identity's name. It never
|
|
// drains or reconciles -- those are store-wide and run in the shared
|
|
// maintainAndDrain loop.
|
|
func (d *Daemon) pollForIdentity(ctx context.Context, id *IdentityRunner) {
|
|
cfg := configForIdentity(d.Cfg.Get(), id.Cfg)
|
|
for _, repo := range id.Repos {
|
|
issues := d.pollIssuesWithConfig(ctx, id.Forge, cfg, repo)
|
|
for _, is := range issues {
|
|
labels := strings.Join(is.Labels, ",")
|
|
if d.Tasks != nil {
|
|
d.pollNATS(ctx, id.Forge, cfg, repo, is, labels, id.Name)
|
|
} else {
|
|
d.pollSQLite(ctx, id.Forge, cfg, repo, is, labels, id.Name)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// maintainAndDrain runs the store-wide passes of a multi-identity cycle:
|
|
// storage cleanup, PR reconciliation, then task dispatch. A single shared
|
|
// loop runs this so the global concurrency cap and per-repo serialization
|
|
// hold across identities.
|
|
func (d *Daemon) maintainAndDrain(ctx context.Context) {
|
|
d.cleanupExpiredStorage(ctx)
|
|
d.reconcilePRs(ctx)
|
|
d.dispatchBindings(ctx)
|
|
if d.Tasks != nil {
|
|
d.drainNATS(ctx)
|
|
} else {
|
|
d.drainSQLite(ctx)
|
|
}
|
|
}
|
|
|
|
// pollIssuesWithConfig discovers work for one repo using the given forge
|
|
// client and dispatch config. Single-identity mode passes d.Forge/d.Cfg;
|
|
// identity poll loops pass the identity's own forge and configForIdentity,
|
|
// so each identity polls with its own bot user, label, and trigger.
|
|
func (d *Daemon) pollIssuesWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) []forge.Issue {
|
|
switch cfg.Dispatch.Trigger {
|
|
case "label":
|
|
// Empty-label rule shared with workintake.MatchesDispatch: a label
|
|
// trigger needs a non-empty label or it would match every open issue.
|
|
if workintake.RequiresLabel(cfg.Dispatch.Trigger) && cfg.Label == "" {
|
|
d.Log.Error("label trigger configured with an empty label; refusing to poll (an empty label matches every open issue)", "repo", repo.FullName())
|
|
return nil
|
|
}
|
|
issues, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
|
|
if err != nil {
|
|
d.Log.Error("label poll failed", "repo", repo.FullName(), "err", err)
|
|
return nil
|
|
}
|
|
return issues
|
|
case "either":
|
|
return d.pollEitherWithConfig(ctx, fg, cfg, repo)
|
|
default: // "assignee" (default)
|
|
issues, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
|
|
if err != nil {
|
|
d.Log.Error("poll failed", "repo", repo.FullName(), "err", err)
|
|
return nil
|
|
}
|
|
return issues
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) pollEitherWithConfig(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo) []forge.Issue {
|
|
seen := map[int]bool{}
|
|
var out []forge.Issue
|
|
|
|
assigned, err := fg.AssignedIssues(ctx, repo.Owner, repo.Name, cfg.BotUser)
|
|
if err != nil {
|
|
d.Log.Error("assigned poll failed", "repo", repo.FullName(), "err", err)
|
|
} else {
|
|
for _, is := range assigned {
|
|
seen[is.Number] = true
|
|
out = append(out, is)
|
|
}
|
|
}
|
|
if workintake.RequiresLabel(cfg.Dispatch.Trigger) && cfg.Label == "" {
|
|
d.Log.Error("either trigger configured with an empty label; skipping the label poll (an empty label matches every open issue)", "repo", repo.FullName())
|
|
return out
|
|
}
|
|
labelled, err := fg.IssuesWithLabel(ctx, repo.Owner, repo.Name, cfg.Label)
|
|
if err != nil {
|
|
d.Log.Error("label poll failed", "repo", repo.FullName(), "err", err)
|
|
} else {
|
|
for _, is := range labelled {
|
|
if !seen[is.Number] {
|
|
out = append(out, is)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Cycle is one poll-and-drain pass: enqueue newly assigned issues,
|
|
// reconcile open PRs, run the binding dispatch loop, then process
|
|
// queued tasks concurrently up to containers.max_concurrency.
|
|
func (d *Daemon) Cycle(ctx context.Context) {
|
|
d.cleanupExpiredStorage(ctx)
|
|
d.poll(ctx)
|
|
d.reconcilePRs(ctx)
|
|
d.dispatchBindings(ctx)
|
|
if d.Tasks != nil {
|
|
d.drainNATS(ctx)
|
|
} else {
|
|
d.drainSQLite(ctx)
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) {
|
|
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
|
|
if ttl <= 0 {
|
|
return
|
|
}
|
|
if d.Storage != nil {
|
|
n, err := d.Storage.CleanupExpired(ctx, ttl)
|
|
if err != nil {
|
|
d.Log.Warn("persistent volume cleanup failed", "err", err)
|
|
}
|
|
if n > 0 {
|
|
d.Log.Info("expired persistent volumes removed", "count", n)
|
|
}
|
|
}
|
|
}
|
|
|
|
// drainSQLite claims queued tasks from SQLite and dispatches them concurrently.
|
|
func (d *Daemon) drainSQLite(ctx context.Context) {
|
|
dispatcher := newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask)
|
|
for ctx.Err() == nil {
|
|
task, err := d.Store.ClaimNext(ctx)
|
|
if err != nil {
|
|
d.Log.Error("claim failed", "err", err)
|
|
break
|
|
}
|
|
if task == nil {
|
|
break
|
|
}
|
|
dispatcher.Submit(ctx, task, d.process)
|
|
}
|
|
dispatcher.Wait()
|
|
}
|
|
|
|
// bindingDispatchBatchLimit caps how many undispatched captures a single
|
|
// dispatchBindings pass will fetch from the store. The cap is defensive
|
|
// (the captures table is also pruned by retention/maxEvents) and bounds a
|
|
// single cycle's worst-case latency on a backlog.
|
|
const bindingDispatchBatchLimit = 100
|
|
|
|
// dispatchBindings walks authenticated captures whose source has at
|
|
// least one armed binding, evaluates each binding's matcher against
|
|
// the capture, resolves fields through the binding's mapping, and
|
|
// enqueues a task per matched (binding, capture) pair.
|
|
//
|
|
// The binding_dispatches ledger is the at-most-once guarantee across
|
|
// cycles and daemon restarts: RecordDispatch returns
|
|
// store.ErrAlreadyDispatched on a duplicate (binding_id, capture_id)
|
|
// pair, and dispatchOneBinding treats that as a normal "another cycle
|
|
// raced us" outcome rather than an error.
|
|
//
|
|
// Nil Bindings disables the loop (legacy behaviour: daemons built
|
|
// before t2db.4 Phase E had no playbook bindings).
|
|
func (d *Daemon) dispatchBindings(ctx context.Context) {
|
|
if d.Bindings == nil {
|
|
return
|
|
}
|
|
|
|
// Gather armed sources first so the captures query filters down to
|
|
// only relevant rows -- the table-level view over captured_events
|
|
// is otherwise unbounded, and a backlog of unrelated sources
|
|
// should never be scanned on a cycle.
|
|
bindings, err := d.Bindings.ListBindings(ctx)
|
|
if err != nil {
|
|
d.Log.Warn("list bindings failed", "error", err)
|
|
return
|
|
}
|
|
sources := make([]string, 0, len(bindings))
|
|
for _, b := range bindings {
|
|
if b.Status == binding.StatusArmed && b.Matcher.Source != "" {
|
|
sources = append(sources, b.Matcher.Source)
|
|
}
|
|
}
|
|
if len(sources) == 0 {
|
|
return
|
|
}
|
|
|
|
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, sources, bindingDispatchBatchLimit)
|
|
if err != nil {
|
|
d.Log.Warn("list undispatched captures failed", "error", err)
|
|
return
|
|
}
|
|
for _, c := range captures {
|
|
armed, err := d.BindingDispatcher.ArmedBindingsForSource(ctx, c.Source)
|
|
if err != nil {
|
|
d.Log.Warn("armed bindings lookup", "source", c.Source, "error", err)
|
|
continue
|
|
}
|
|
for _, b := range armed {
|
|
if !b.Matcher.Matches(c.Source, c.Authenticated) {
|
|
continue
|
|
}
|
|
d.dispatchOneBinding(ctx, b, c)
|
|
}
|
|
}
|
|
}
|
|
|
|
// dispatchOneBinding handles a single (binding, capture) match: resolve
|
|
// fields through the binding's mapping, enqueue a task, and record the
|
|
// dispatch in the dedup ledger. Each step degrades independently --
|
|
// a mapping lookup failure logs and returns, a required-field
|
|
// resolution failure records a binding_dispatch_failure event and
|
|
// returns without enqueueing, an enqueue failure logs and returns, and
|
|
// a RecordDispatch returning ErrAlreadyDispatched is treated as a
|
|
// benign race winner (another cycle already created the task).
|
|
//
|
|
// Owner/repo resolution currently falls back to the single configured
|
|
// repo (when exactly one is configured); binding.Owner/Repo is a
|
|
// follow-up so multi-repo deployments don't have to live with the
|
|
// ambiguity.
|
|
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c store.CapturedEvent) {
|
|
if d.Mappings == nil {
|
|
d.Log.Warn("binding dispatch: mapping store unavailable", "binding", b.ID)
|
|
return
|
|
}
|
|
m, err := d.Mappings.GetMapping(ctx, b.MappingID)
|
|
if err != nil {
|
|
d.Log.Warn("binding dispatch: get mapping", "binding", b.ID, "mapping", b.MappingID, "error", err)
|
|
return
|
|
}
|
|
if m == nil {
|
|
d.Log.Warn("binding dispatch: mapping missing", "binding", b.ID, "mapping", b.MappingID)
|
|
return
|
|
}
|
|
values, failures := mapping.Resolve(m.Fields, []byte(c.Body))
|
|
if hasBlockingFailure(m.Fields, failures) {
|
|
_, _ = d.Store.InsertEvent(ctx, events.Event{
|
|
Kind: "binding_dispatch_failure",
|
|
Detail: fmt.Sprintf("binding %d (capture %d): required field failed", b.ID, c.ID),
|
|
Data: map[string]any{
|
|
"binding_id": b.ID,
|
|
"binding_version": b.Version,
|
|
"capture_id": c.ID,
|
|
"failures": failures,
|
|
},
|
|
})
|
|
return
|
|
}
|
|
|
|
owner, repo, ok := d.resolveBindingRepo(b)
|
|
if !ok {
|
|
return // resolveBindingRepo already logged a warning
|
|
}
|
|
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
|
|
body := renderBindingBody(values, c)
|
|
|
|
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, owner, repo, title, body, b.Workflow, "", b.ID, b.Version)
|
|
if err != nil {
|
|
d.Log.Warn("binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
|
|
return
|
|
}
|
|
|
|
if err := d.BindingDispatcher.RecordDispatch(ctx, nil, b.ID, int64(b.Version), c.ID, task.ID); err != nil {
|
|
if errors.Is(err, store.ErrAlreadyDispatched) {
|
|
// Another cycle or daemon instance raced us and already
|
|
// recorded the dispatch. The duplicate task remains
|
|
// queued (delete-on-races is its own can of worms),
|
|
// but the ledger prevents the same (binding, capture)
|
|
// pair from ever being dispatched again.
|
|
return
|
|
}
|
|
d.Log.Warn("binding dispatch: record", "binding", b.ID, "capture", c.ID, "error", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// hasBlockingFailure reports whether any of the failures belongs to a
|
|
// required field. The mapping package reports failures only by field
|
|
// name and reason -- the Field's Required flag is on the Field
|
|
// declaration, so the dispatch loop re-derives it from the full
|
|
// Field list. Failures on optional fields are warnings, not blocks:
|
|
// the workflow still receives a partial value map.
|
|
func hasBlockingFailure(fields []mapping.Field, failures []mapping.Failure) bool {
|
|
if len(failures) == 0 {
|
|
return false
|
|
}
|
|
required := make(map[string]bool, len(fields))
|
|
for _, f := range fields {
|
|
if f.Required {
|
|
required[f.Name] = true
|
|
}
|
|
}
|
|
for _, fail := range failures {
|
|
if required[fail.FieldName] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// renderBindingBody builds the task body from the values a mapping
|
|
// resolved, one "key=value" pair per line. Empty maps still produce
|
|
// a body that records the source capture id so the workflow has
|
|
// provenance to hand back to the operator.
|
|
func renderBindingBody(values map[string]any, c store.CapturedEvent) string {
|
|
if len(values) == 0 {
|
|
return fmt.Sprintf("(no fields resolved from capture %d)", c.ID)
|
|
}
|
|
parts := make([]string, 0, len(values))
|
|
for k, v := range values {
|
|
parts = append(parts, fmt.Sprintf("%s=%v", k, v))
|
|
}
|
|
return strings.Join(parts, "\n")
|
|
}
|
|
|
|
// resolveBindingRepo picks the owner/repo a binding-spawned task
|
|
// operates against. A binding that pins Owner/Repo (binding.Binding's
|
|
// Owner/Repo fields, validated as a complete pair or not set at all)
|
|
// dispatches to that target unconditionally -- multi-repo deployments
|
|
// use this to disambiguate. An unpinned binding falls back to the
|
|
// single-configured-repo behaviour that predates the pin: with exactly
|
|
// one configured repo the choice is unambiguous; with zero or more than
|
|
// one the loop refuses to dispatch (logged below) so a binding cannot
|
|
// silently fire against the wrong target.
|
|
func (d *Daemon) resolveBindingRepo(b binding.Binding) (string, string, bool) {
|
|
if b.Owner != "" && b.Repo != "" {
|
|
return b.Owner, b.Repo, true
|
|
}
|
|
repos := d.Cfg.Get().Repos
|
|
if len(repos) == 1 {
|
|
return repos[0].Owner, repos[0].Name, true
|
|
}
|
|
d.Log.Warn("binding dispatch: cannot resolve target repo",
|
|
"hint", "configure exactly one [[repos]] entry, or pin the binding to a specific owner/repo",
|
|
"configured_repos", len(repos))
|
|
return "", "", false
|
|
}
|
|
|
|
// drainNATS processes tasks from NATS, falling back to SQLite ClaimNext for
|
|
// requeued tasks (waiting_human approval, retry-parked) that didn't come
|
|
// through a NATS publish.
|
|
func (d *Daemon) drainNATS(ctx context.Context) {
|
|
dispatcher := newTaskDispatcher(d.Cfg.Get().Containers.MaxConcurrency, d.allowConcurrentForTask)
|
|
for ctx.Err() == nil {
|
|
msg, err := d.Tasks.Fetch(ctx)
|
|
if err != nil && !errors.Is(err, eventbus.ErrNoMessage) {
|
|
d.Log.Error("nats fetch failed", "err", err)
|
|
break
|
|
}
|
|
if err == nil {
|
|
d.submitNATSTask(ctx, dispatcher, msg)
|
|
continue
|
|
}
|
|
|
|
// ErrNoMessage means the stream is drained. That is precisely when
|
|
// the SQLite fallback must run: requeued tasks (waiting_human
|
|
// approval, retry-parked) never went through a NATS publish, so
|
|
// nothing will ever arrive on the stream to trigger them.
|
|
task, claimErr := d.Store.ClaimNext(ctx)
|
|
if claimErr != nil {
|
|
d.Log.Error("sqlite claim failed", "err", claimErr)
|
|
break
|
|
}
|
|
if task == nil {
|
|
break
|
|
}
|
|
dispatcher.Submit(ctx, task, d.process)
|
|
}
|
|
dispatcher.Wait()
|
|
}
|
|
|
|
// submitNATSTask decodes a fetched message and queues it for processing. A
|
|
// message that cannot be decoded is acked rather than redelivered: it will
|
|
// never decode on a retry, so leaving it unacked would block the queue.
|
|
func (d *Daemon) submitNATSTask(ctx context.Context, dispatcher *taskDispatcher, msg eventbus.Message) {
|
|
envelope, err := workintake.DecodeTask(msg.Data())
|
|
if err != nil {
|
|
d.Log.Error("nats decode failed", "subject", msg.Subject(), "err", err)
|
|
_ = msg.Ack()
|
|
return
|
|
}
|
|
task := &store.Task{Owner: envelope.Owner, Repo: envelope.Repo, Identity: envelope.Identity}
|
|
dispatcher.Submit(ctx, task, func(ctx context.Context, _ *store.Task) {
|
|
d.processNATSTask(ctx, msg)
|
|
})
|
|
}
|
|
|
|
// taskDispatcher bounds task execution globally while preserving the default
|
|
// one-task-per-repository safety rule. Submit order is retained within a repo;
|
|
// tasks for different repos may run concurrently. Repos for which
|
|
// allowConcurrent reports true opt out of the same-repo serialization
|
|
// entirely -- the global slot limit is the only bound on their concurrency.
|
|
type taskDispatcher struct {
|
|
slots chan struct{}
|
|
allowConcurrent func(task *store.Task) bool
|
|
|
|
mu sync.Mutex
|
|
repoTail map[string]chan struct{}
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
func newTaskDispatcher(maxConcurrency int, allowConcurrent func(task *store.Task) bool) *taskDispatcher {
|
|
var slots chan struct{}
|
|
if maxConcurrency > 0 {
|
|
slots = make(chan struct{}, maxConcurrency)
|
|
}
|
|
if allowConcurrent == nil {
|
|
allowConcurrent = func(*store.Task) bool { return false }
|
|
}
|
|
return &taskDispatcher{
|
|
slots: slots,
|
|
allowConcurrent: allowConcurrent,
|
|
repoTail: make(map[string]chan struct{}),
|
|
}
|
|
}
|
|
|
|
func (d *taskDispatcher) Submit(
|
|
ctx context.Context,
|
|
task *store.Task,
|
|
process func(context.Context, *store.Task),
|
|
) {
|
|
repo := task.Owner + "/" + task.Repo
|
|
|
|
var previous, done chan struct{}
|
|
if !d.allowConcurrent(task) {
|
|
done = make(chan struct{})
|
|
d.mu.Lock()
|
|
previous = d.repoTail[repo]
|
|
d.repoTail[repo] = done
|
|
d.mu.Unlock()
|
|
}
|
|
|
|
d.wg.Go(func() {
|
|
if previous != nil {
|
|
<-previous
|
|
}
|
|
if d.slots != nil {
|
|
d.slots <- struct{}{}
|
|
defer func() { <-d.slots }()
|
|
}
|
|
if done != nil {
|
|
defer func() {
|
|
close(done)
|
|
d.mu.Lock()
|
|
if d.repoTail[repo] == done {
|
|
delete(d.repoTail, repo)
|
|
}
|
|
d.mu.Unlock()
|
|
}()
|
|
}
|
|
process(ctx, task)
|
|
})
|
|
}
|
|
|
|
func (d *taskDispatcher) Wait() {
|
|
d.wg.Wait()
|
|
}
|
|
|
|
func (d *Daemon) poll(ctx context.Context) {
|
|
for _, repo := range d.Cfg.Get().Repos {
|
|
issues := d.pollIssues(ctx, repo)
|
|
for _, is := range issues {
|
|
labels := strings.Join(is.Labels, ",")
|
|
if d.Tasks != nil {
|
|
d.pollNATS(ctx, d.Forge, d.Cfg.Get(), repo, is, labels, "")
|
|
} else {
|
|
d.pollSQLite(ctx, d.Forge, d.Cfg.Get(), repo, is, labels, "")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// pollSQLite enqueues discovered issues directly into SQLite (existing flow).
|
|
// fg is the forge client that discovered is -- d.Forge for the
|
|
// single-identity path, or an identity's own client from pollForIdentity.
|
|
// cfg is the dispatch config that owns this poll (root or identity-scoped).
|
|
// identity records which identity owns the resulting task; empty for
|
|
// single-identity deployments.
|
|
func (d *Daemon) pollSQLite(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) {
|
|
inserted, err := d.Store.EnqueueIssue(ctx,
|
|
repo.Owner, repo.Name, is.Number, is.Title, is.Body, labels, identity)
|
|
if err != nil {
|
|
d.Log.Error("enqueue failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
|
return
|
|
}
|
|
if inserted {
|
|
d.acknowledge(ctx, fg, cfg, repo, is)
|
|
return
|
|
}
|
|
// Existing tasks are left untouched. Retry and approval are explicit
|
|
// operator actions in messaging or the Web UI, not forge-side label edits.
|
|
}
|
|
|
|
// pollNATS publishes discovered issues to NATS (new flow).
|
|
func (d *Daemon) pollNATS(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue, labels, identity string) {
|
|
// Read-only existence check prevents rediscovering an existing task.
|
|
existing, err := d.Store.TaskByIssue(ctx, repo.Owner, repo.Name, is.Number)
|
|
if err != nil {
|
|
d.Log.Error("task lookup failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
|
return
|
|
}
|
|
if existing != nil {
|
|
return
|
|
}
|
|
// Classify and encode here: the envelope, its routing kind and its
|
|
// subject belong to the work-intake domain, not to the transport.
|
|
parsed := workintake.SplitLabels(labels)
|
|
task := workintake.TaskEnvelope{
|
|
Owner: repo.Owner,
|
|
Repo: repo.Name,
|
|
Number: is.Number,
|
|
Title: is.Title,
|
|
Body: is.Body,
|
|
Labels: parsed,
|
|
Identity: identity,
|
|
Kind: workintake.KindForLabels(parsed),
|
|
}
|
|
if err := d.PublishTask(ctx, task); err != nil {
|
|
d.Log.Error("task publish failed", "repo", repo.FullName(), "issue", is.Number, "err", err)
|
|
return
|
|
}
|
|
d.acknowledge(ctx, fg, cfg, repo, is)
|
|
}
|
|
|
|
// PublishTask validates, encodes and publishes a task envelope. The
|
|
// idempotency key means rediscovering the same issue on a later poll does not
|
|
// enqueue the work twice. It is the single enqueue path shared by the poller
|
|
// and the forge webhook intake, so neither can drift in how a discovered issue
|
|
// becomes a task.
|
|
func (d *Daemon) PublishTask(ctx context.Context, task workintake.TaskEnvelope) error {
|
|
if d.Tasks == nil {
|
|
return fmt.Errorf("publish task %s: no task bus configured", task.Ref())
|
|
}
|
|
if err := task.Kind.Validate(); err != nil {
|
|
return fmt.Errorf("publish task %s: %w", task.Ref(), err)
|
|
}
|
|
payload, err := task.Encode()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return d.Tasks.PublishUnique(ctx, task.Subject(), task.IdempotencyKey(), payload)
|
|
}
|
|
|
|
// acknowledge posts the pickup reaction and queued event
|
|
// using fg -- the forge client that owns repo (identity-scoped or root) --
|
|
// and the ack reaction from cfg, the dispatch config that owns this poll.
|
|
func (d *Daemon) acknowledge(ctx context.Context, fg forge.Forge, cfg config.Config, repo config.Repo, is forge.Issue) {
|
|
d.Log.Info("issue queued", "repo", repo.FullName(), "issue", is.Number, "title", is.Title)
|
|
if ack := cfg.Dispatch.AckReaction; ack != "" {
|
|
if err := fg.React(ctx, repo.Owner, repo.Name, is.Number, ack); err != nil {
|
|
d.Log.Warn("ack reaction failed", "issue", is.Number, "err", err)
|
|
}
|
|
}
|
|
d.emit(events.Event{
|
|
Kind: events.KindTaskQueued, Repo: repo.FullName(),
|
|
Issue: is.Number, Detail: is.Title,
|
|
})
|
|
}
|
|
|
|
// processNATSTask decodes a NATS message, writes it to SQLite, claims it,
|
|
// and runs the workflow. The message is ack'd on terminal (park is a valid
|
|
// outcome). Nak on transient errors so NATS redelivers.
|
|
func (d *Daemon) processNATSTask(ctx context.Context, msg eventbus.Message) {
|
|
tm, err := workintake.DecodeTask(msg.Data())
|
|
if err != nil {
|
|
d.Log.Error("nats decode failed", "err", err)
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
// bad message, don't retry
|
|
return
|
|
}
|
|
|
|
inserted, err := d.Store.EnqueueIssue(ctx, tm.Owner, tm.Repo, tm.Number, tm.Title, tm.Body, strings.Join(tm.Labels, ","), tm.Identity)
|
|
if err != nil {
|
|
d.Log.Error("nats enqueue failed", "err", err)
|
|
if err := msg.Nak(); err != nil {
|
|
d.Log.Warn("nats nak failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
if !inserted {
|
|
// Already tracked -- dedup. Ack and move on.
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
task, err := d.Store.ClaimByIssue(ctx, tm.Owner, tm.Repo, tm.Number)
|
|
if err != nil {
|
|
d.Log.Error("nats claim failed", "err", err)
|
|
if err := msg.Nak(); err != nil {
|
|
d.Log.Warn("nats nak failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
if task == nil {
|
|
// Claimed by another consumer, or task is not queued. Ack.
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
d.process(ctx, task)
|
|
if err := msg.Ack(); err != nil {
|
|
d.Log.Warn("ack failed", "err", err)
|
|
}
|
|
}
|
|
|
|
// pollIssues discovers work for one repo according to the root dispatch
|
|
// config, using the root forge client. Identity poll loops use
|
|
// pollIssuesWithConfig with their own forge and config instead.
|
|
func (d *Daemon) pollIssues(ctx context.Context, repo config.Repo) []forge.Issue {
|
|
return d.pollIssuesWithConfig(ctx, d.Forge, d.Cfg.Get(), repo)
|
|
}
|
|
|
|
// closeResolvedIssue closes the forge issue behind a finished task.
|
|
//
|
|
// A chat task's issue number is synthetic and matches no forge issue, so it
|
|
// is skipped. Failure is logged rather than returned: the task's own state is
|
|
// already correct, and the next reconcile pass will not retry, so a warning
|
|
// is the honest outcome -- the issue simply stays open.
|
|
func (d *Daemon) closeResolvedIssue(ctx context.Context, fg forge.Forge, task *store.Task, comment string) {
|
|
if !task.IsForgeBacked() {
|
|
return
|
|
}
|
|
if err := fg.CloseIssue(ctx, task.Owner, task.Repo, task.IssueNumber, comment); err != nil {
|
|
d.Log.Warn("closing resolved issue failed; it stays open and will be re-polled",
|
|
"repo", task.Owner+"/"+task.Repo, "issue", task.IssueNumber, "err", err)
|
|
}
|
|
}
|
|
|
|
// reconcilePRs moves pr_open tasks to merged/rejected from GitHub state
|
|
// and cleans up their worktrees.
|
|
func (d *Daemon) reconcilePRs(ctx context.Context) {
|
|
tasks, err := d.Store.OpenPRs(ctx)
|
|
if err != nil {
|
|
d.Log.Error("reconcile query failed", "err", err)
|
|
return
|
|
}
|
|
for _, t := range tasks {
|
|
fg := d.forgeFor(&t)
|
|
trees := d.treesFor(&t)
|
|
// PRState/PR reconciliation is valid regardless of Source -- a
|
|
// chat-sourced task can still open a real PR against a real
|
|
// branch. Only the issue-label call (tied to the synthetic
|
|
// issue number) is forge-only.
|
|
state, err := fg.PRState(ctx, t.Owner, t.Repo, t.PRNumber)
|
|
if err != nil {
|
|
d.Log.Warn("PR state check failed", "pr", t.PRNumber, "err", err)
|
|
continue
|
|
}
|
|
switch state {
|
|
case "merged":
|
|
_ = d.Store.Transition(ctx, t.ID, store.StatusPROpen, store.StatusMerged, "")
|
|
_ = trees.Cleanup(t.Owner, t.Repo, t.IssueNumber)
|
|
// Close the issue ourselves rather than relying on the forge
|
|
// noticing a "Closes #N" in the PR body. Nothing else closes it:
|
|
// LinkBranch is sidebar linkage on Gitea and a no-op on GitHub.
|
|
// Left open the issue stays labelled and assigned, the next poll
|
|
// re-enqueues it, and once the dashboard's Clear removes the task
|
|
// row that is a second implementation and a second PR for work
|
|
// already merged.
|
|
d.closeResolvedIssue(ctx, fg, &t, fmt.Sprintf(
|
|
"Closed by #%d, merged by archie.", t.PRNumber,
|
|
))
|
|
d.Log.Info("PR merged", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
|
|
d.emit(events.Event{
|
|
Kind: events.KindPRMerged, TaskID: t.ID,
|
|
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
|
|
Data: map[string]any{"pr": t.PRNumber},
|
|
})
|
|
case "closed":
|
|
_ = d.Store.Transition(ctx, t.ID, store.StatusPROpen, store.StatusRejected, "PR closed without merge")
|
|
_ = trees.Cleanup(t.Owner, t.Repo, t.IssueNumber)
|
|
d.Log.Info("PR rejected", "repo", t.Owner+"/"+t.Repo, "pr", t.PRNumber)
|
|
d.emit(events.Event{
|
|
Kind: events.KindPRRejected, TaskID: t.ID,
|
|
Repo: t.Owner + "/" + t.Repo, Issue: t.IssueNumber,
|
|
Data: map[string]any{"pr": t.PRNumber},
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) process(ctx context.Context, task *store.Task) {
|
|
// Register before any work starts so the task is stoppable for its
|
|
// whole life, including the slow setup -- clone, worktree prepare,
|
|
// image pull -- which is exactly when someone realises they asked for
|
|
// the wrong thing. Every caller reaches execution through here, so
|
|
// this is the one place that has to do it.
|
|
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
|
defer finished()
|
|
defer d.openTaskLog(task)()
|
|
|
|
trees := d.treesFor(task)
|
|
repo, ok := d.repoFor(task)
|
|
if !ok {
|
|
d.parkRunningTask(ctx, task.ID, "repo no longer in config")
|
|
return
|
|
}
|
|
if d.ContainerPool == nil {
|
|
const reason = "managed agent container pool is unavailable; refusing to run this task on the host"
|
|
d.Log.Error("task parked: managed worker unavailable", "task", task.ID,
|
|
"hint", "enable containers and make the archie-agent image available, then retry the task")
|
|
if err := d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason); err != nil {
|
|
d.Log.Warn("managed worker park transition failed", "task", task.ID, "err", err)
|
|
return
|
|
}
|
|
d.emit(events.Event{
|
|
Kind: events.KindParked, TaskID: task.ID,
|
|
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
|
|
})
|
|
return
|
|
}
|
|
if d.Tasks == nil {
|
|
const reason = "agent task transport is unavailable; refusing to run this task on the host"
|
|
d.Log.Error("task parked: agent task transport unavailable", "task", task.ID)
|
|
if err := d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason); err != nil {
|
|
d.Log.Warn("agent task transport park transition failed", "task", task.ID, "err", err)
|
|
return
|
|
}
|
|
d.emit(events.Event{
|
|
Kind: events.KindParked, TaskID: task.ID,
|
|
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
|
|
})
|
|
return
|
|
}
|
|
|
|
workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
|
|
d.Log.Info("processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
|
|
|
|
// Clone the worktree before Docker mount setup. The container
|
|
// binds /data/worktree to workDir on the host -- the directory
|
|
// must exist before Acquire is called.
|
|
var branch string
|
|
var err error
|
|
// Every task gets an independent full clone. The former
|
|
// PreparePersistent path shared objects with a per-repo bare cache;
|
|
// go-git has no --dissociate, so a shared cache would stay a live
|
|
// dependency of each worktree and expiring one would corrupt running
|
|
// tasks. repo.PersistentStorage still governs the container volume.
|
|
_, branch, err = trees.Prepare(ctx, task.Owner, task.Repo, repo.Base, task.IssueNumber, task.Title, task.Body, task.Labels)
|
|
if err != nil {
|
|
d.Log.Error("worktree prepare failed", "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "worktree prepare failed: "+err.Error())
|
|
return
|
|
}
|
|
task.Branch = branch
|
|
if err := d.Store.Update(ctx, task); err != nil {
|
|
d.Log.Warn("task branch not persisted", "task", task.ID, "err", err)
|
|
}
|
|
|
|
ctr, ok := d.acquireTaskContainer(ctx, task, repo, workDir)
|
|
if !ok {
|
|
return
|
|
}
|
|
defer d.ContainerPool.Release(ctx, ctr)
|
|
|
|
// Hand the whole task to archie-agent in one NATS round trip. archie-agent
|
|
// proxies Store/Forge/worktree-push calls back to archied over storerpc/
|
|
// forgerpc/worktreerpc -- by the time runViaAgent returns, the task's
|
|
// terminal state already landed via those calls. The task's identity
|
|
// travels in the taskrun request; the agent scopes its forgerpc and
|
|
// worktreerpc clients to that identity, and the daemon registered one
|
|
// server pair per identity (plus the root pair), so a container-mode
|
|
// task is always served by its own forge client and worktree manager.
|
|
d.runViaAgent(ctx, task, repo)
|
|
|
|
// Teardown storage after workflow completes. The Docker backend is a
|
|
// no-op; future backends (temp volumes, NFS leases) use this hook.
|
|
if d.Storage != nil {
|
|
_ = d.Storage.Teardown(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
})
|
|
}
|
|
|
|
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
|
}
|
|
|
|
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *store.Task, trees *worktree.Manager) {
|
|
if d.Store == nil || trees == nil || task == nil {
|
|
return
|
|
}
|
|
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
|
|
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
|
|
if err != nil {
|
|
d.Log.Warn("terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
|
|
return
|
|
}
|
|
if latest == nil {
|
|
return
|
|
}
|
|
switch latest.Status {
|
|
case store.StatusMerged, store.StatusRejected, store.StatusClosedWontDo:
|
|
if latest.PRNumber == 0 {
|
|
if err := trees.Cleanup(task.Owner, task.Repo, task.IssueNumber); err != nil {
|
|
d.Log.Warn("terminal worktree cleanup failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// openTaskLog opens task's log sink for the lifetime of one process() call
|
|
// and returns the function to close it. That lifetime covers preparation,
|
|
// the full-task container handoff, and teardown. A failure to open is not
|
|
// fatal to the task -- it means this
|
|
// attempt's own output won't be recoverable if something goes wrong, which
|
|
// is worse than the pre-existing state, not equal to it, so the run
|
|
// proceeds rather than parking over a logging problem.
|
|
func (d *Daemon) openTaskLog(task *store.Task) func() {
|
|
if err := d.TaskLogs.Open(task.ID, task.Attempt); err != nil {
|
|
d.Log.Warn("task log sink unavailable", "task", task.ID, "attempt", task.Attempt, "err", err)
|
|
}
|
|
return func() {
|
|
if err := d.TaskLogs.Close(task.ID); err != nil {
|
|
d.Log.Warn("task log sink close failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// acquireTaskContainer prepares and acquires the sandbox container for a
|
|
// task, reporting false when it could not -- having already parked the
|
|
// task and logged why. The caller releases the container it returns.
|
|
func (d *Daemon) acquireTaskContainer(
|
|
ctx context.Context,
|
|
task *store.Task,
|
|
repo config.Repo,
|
|
workDir string,
|
|
) (*container.Container, bool) {
|
|
park := func(reason string, err error) {
|
|
d.Log.Error(reason, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, reason+": "+err.Error())
|
|
}
|
|
|
|
// Write task.json -- the container's boot-time brief.
|
|
if err := container.WriteTaskJSON(workDir, container.TaskPayload{
|
|
ID: task.ID, Owner: task.Owner, Repo: task.Repo,
|
|
Number: task.IssueNumber, Title: task.Title, Body: task.Body,
|
|
Labels: strings.Split(task.Labels, ","),
|
|
Workflow: task.Workflow, Branch: task.Branch, Plan: task.Plan,
|
|
}); err != nil {
|
|
park("task.json write failed", err)
|
|
return nil, false
|
|
}
|
|
|
|
// Guard: Storage may be nil if the daemon was wired incorrectly. In
|
|
// normal operation, Storage is always set when ContainerPool is set.
|
|
if d.Storage == nil {
|
|
d.Log.Error("storage backend is nil -- cannot acquire container")
|
|
d.parkRunningTask(ctx, task.ID, "storage backend not configured")
|
|
return nil, false
|
|
}
|
|
|
|
mounts, err := d.Storage.Setup(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
})
|
|
if err != nil {
|
|
park("storage setup failed", err)
|
|
return nil, false
|
|
}
|
|
|
|
ctr, err := d.ContainerPool.Acquire(ctx, mounts, d.containerEnv(task))
|
|
if err != nil {
|
|
// Roll back the storage we just set up: the mounts were created
|
|
// for this task but no container will use them. Leaking them until
|
|
// a later TTL sweep is not acceptable on a backend that allocates
|
|
// real resources (volumes, NFS leases).
|
|
if terr := d.Storage.Teardown(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
}); terr != nil {
|
|
d.Log.Warn("storage teardown after acquire failure failed", "err", terr)
|
|
}
|
|
park("container acquire failed", err)
|
|
return nil, false
|
|
}
|
|
return ctr, true
|
|
}
|
|
|
|
// parkRunningTask transitions a task from running to parked using a context
|
|
// detached from caller cancellation, with a bounded timeout.
|
|
//
|
|
// When a task fails or is cancelled (e.g. via /stop or dashboard Stop), ctx is
|
|
// already cancelled. Writing terminal state using the cancelled ctx fails
|
|
// immediately in SQLite transactions and silently leaves the task row 'running'
|
|
// indefinitely. Like container teardown in Pool.Release, terminal state
|
|
// recording is cleanup on the way out of a task and must use a bounded context
|
|
// that survives cancellation while preserving values.
|
|
//
|
|
// The transition remains guarded from StatusRunning: if the worker already
|
|
// recorded a terminal state over storerpc, store.ErrStaleTransition is returned
|
|
// and ignored. Unexpected store errors are logged as warnings.
|
|
func (d *Daemon) parkRunningTask(ctx context.Context, taskID int64, reason string) {
|
|
if d.Store == nil {
|
|
return
|
|
}
|
|
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
|
|
if err := d.Store.Transition(writeCtx, taskID, store.StatusRunning, store.StatusParked, reason); err != nil {
|
|
if !errors.Is(err, store.ErrStaleTransition) {
|
|
d.Log.Warn("terminal park transition failed", "task", taskID, "reason", reason, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// containerEnv returns the environment variables passed to agent containers.
|
|
// runViaAgent publishes a full-task handoff to archie-agent over the
|
|
// infrastructure-owned task subject and waits for its completion report. Every durable
|
|
// side effect (Store transitions, Forge calls, git push) happens inside
|
|
// archie-agent's workflow.Run, proxied back to this daemon over
|
|
// storerpc/forgerpc/worktreerpc -- those RPC servers are the sole place a
|
|
// terminal state gets written on success. runViaAgent only parks the task
|
|
// itself when nothing else could have: the request never reached (or was
|
|
// never answered by) an archie-agent, or archie-agent failed before its
|
|
// own workflow.Run got a chance to record an outcome.
|
|
func (d *Daemon) runViaAgent(ctx context.Context, task *store.Task, repo config.Repo) {
|
|
if d.WorktreeGrants == nil {
|
|
const reason = "worktree publication grants are unavailable"
|
|
d.Log.Error(reason, "task", task.ID)
|
|
d.parkRunningTask(ctx, task.ID, reason)
|
|
return
|
|
}
|
|
grant, revoke, err := d.WorktreeGrants.Issue(task)
|
|
if err != nil {
|
|
d.Log.Error("worktree publication grant failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "worktree publication grant failed: "+err.Error())
|
|
return
|
|
}
|
|
defer revoke()
|
|
req := taskrun.Request{
|
|
Task: task,
|
|
Repo: repo,
|
|
Cfg: d.configFor(task).ForTask(),
|
|
Providers: agentexec.ProvidersFromConfig(d.configFor(task).Providers),
|
|
WorktreeGrant: grant,
|
|
}
|
|
data, err := json.Marshal(req)
|
|
if err != nil {
|
|
d.Log.Error("taskrun encode failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun encode failed: "+err.Error())
|
|
return
|
|
}
|
|
|
|
reply, err := d.requestTaskRun(ctx, task.ID, data)
|
|
if err != nil {
|
|
d.Log.Error("taskrun request failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun request failed: "+err.Error())
|
|
return
|
|
}
|
|
|
|
var resp taskrun.Response
|
|
if err := json.Unmarshal(reply, &resp); err != nil {
|
|
d.Log.Error("taskrun decode response failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun decode response failed: "+err.Error())
|
|
return
|
|
}
|
|
if resp.Error != "" {
|
|
d.Log.Error("taskrun run failed", "task", task.ID, "err", resp.Error)
|
|
d.parkRunningTask(ctx, task.ID, "taskrun run failed: "+resp.Error)
|
|
return
|
|
}
|
|
|
|
// Reached only past every failure return above, so this is always a
|
|
// fully decoded, error-free response -- a version is never recorded off
|
|
// a request that never reached an agent, or a response describing a
|
|
// parked/errored task.
|
|
if d.AgentStatus != nil && resp.AgentVersion != "" {
|
|
d.AgentStatus.Observe(resp.AgentVersion, resp.AgentInstallType)
|
|
}
|
|
|
|
d.Log.Info("taskrun complete", "task", task.ID, "status", resp.Status)
|
|
}
|
|
|
|
const (
|
|
// defaultTaskRunReadyTimeout is how long requestTaskRun retries a
|
|
// nats.ErrNoResponders before giving up, when Daemon.TaskRunReadyTimeout
|
|
// is unset.
|
|
defaultTaskRunReadyTimeout = 20 * time.Second
|
|
// defaultTaskRunRetryBackoff is the delay between retry attempts, when
|
|
// Daemon.TaskRunRetryBackoff is unset.
|
|
defaultTaskRunRetryBackoff = 250 * time.Millisecond
|
|
)
|
|
|
|
func (d *Daemon) taskRunReadyTimeout() time.Duration {
|
|
if d.TaskRunReadyTimeout > 0 {
|
|
return d.TaskRunReadyTimeout
|
|
}
|
|
return defaultTaskRunReadyTimeout
|
|
}
|
|
|
|
func (d *Daemon) taskRunRetryBackoff() time.Duration {
|
|
if d.TaskRunRetryBackoff > 0 {
|
|
return d.TaskRunRetryBackoff
|
|
}
|
|
return defaultTaskRunRetryBackoff
|
|
}
|
|
|
|
// requestTaskRun publishes the taskrun request and retries while no
|
|
// archie-agent has subscribed yet (nats.ErrNoResponders): the container
|
|
// pool's Acquire returns as soon as Docker has issued the start syscall,
|
|
// not once the spawned container has connected to core NATS and subscribed
|
|
// to this per-task subject --
|
|
// a gap of hundreds of milliseconds to a few seconds that would otherwise
|
|
// fail the very first request deterministically, every time, on every
|
|
// task. Any error other than ErrNoResponders (encode failures, a context
|
|
// that's already done, etc.) is returned immediately without retrying,
|
|
// since those don't mean "not ready yet".
|
|
func (d *Daemon) requestTaskRun(ctx context.Context, taskID int64, data []byte) ([]byte, error) {
|
|
subject := agentnats.SubjectForTask(taskID)
|
|
deadline := time.Now().Add(d.taskRunReadyTimeout())
|
|
backoff := d.taskRunRetryBackoff()
|
|
for {
|
|
reply, err := d.Tasks.Request(ctx, subject, data)
|
|
if err == nil {
|
|
return reply, nil
|
|
}
|
|
if !errors.Is(err, eventbus.ErrNoResponders) || !time.Now().Before(deadline) {
|
|
return nil, err
|
|
}
|
|
wait := min(backoff, time.Until(deadline))
|
|
if wait <= 0 {
|
|
return nil, err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-time.After(wait):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) containerEnv(task *store.Task) []string {
|
|
var env []string
|
|
// The endpoint the daemon's own client connected with at startup, not
|
|
// the live config: a reloaded [nats] section must not point new
|
|
// containers at a server the daemon is not publishing on.
|
|
env = append(env, "NATS_URL="+d.ConnectedNATS.URL)
|
|
if token := d.ConnectedNATS.Token; token != "" {
|
|
env = append(env, "NATS_TOKEN="+token)
|
|
}
|
|
// The agent process runs as root inside the container (no USER in the
|
|
// Dockerfile, no userns-remap), so a commit it writes to the bind-mounted
|
|
// worktree lands owned by UID 0 on the host. The daemon then reads those
|
|
// same loose objects to push, running as its own non-root host user --
|
|
// archie-core#520's "permission denied" on .git/objects. Passing the
|
|
// daemon's own UID/GID lets the agent chown the worktree back to it
|
|
// before pushing.
|
|
env = append(env, fmt.Sprintf("WORKTREE_UID=%d", os.Getuid()))
|
|
env = append(env, fmt.Sprintf("WORKTREE_GID=%d", os.Getgid()))
|
|
for _, p := range d.configFor(task).Providers {
|
|
if p.APIKeyEnv != "" {
|
|
if v := os.Getenv(p.APIKeyEnv); v != "" {
|
|
env = append(env, p.APIKeyEnv+"="+v)
|
|
}
|
|
}
|
|
}
|
|
return env
|
|
}
|
|
|
|
func (d *Daemon) configFor(task *store.Task) config.Config {
|
|
if id := d.identityFor(task); id != nil {
|
|
return configForIdentity(d.Cfg.Get(), id.Cfg)
|
|
}
|
|
return d.Cfg.Get()
|
|
}
|
|
|
|
func configForIdentity(root config.Config, identity config.IdentityConfig) config.Config {
|
|
root.BotUser = identity.BotUser
|
|
root.BotEmail = identity.BotEmail
|
|
root.DiffCapLines = identity.DiffCapLines
|
|
root.Forge = identity.Forge
|
|
root.Dispatch = identity.Dispatch
|
|
root.Models = identity.Models
|
|
root.Providers = identity.Providers
|
|
root.Budgets = identity.Budgets
|
|
root.Notify = identity.Notify
|
|
root.Repos = identity.Repos
|
|
return root
|
|
}
|
|
|
|
// identityFor resolves the IdentityRunner that owns task, or nil for
|
|
// single-identity deployments and forge-sourced tasks recorded
|
|
// before multi-identity routing existed (task.Identity == ""). Callers
|
|
// must fall back to the root d.Forge/d.Trees/d.Cfg when this returns nil.
|
|
func (d *Daemon) identityFor(task *store.Task) *IdentityRunner {
|
|
if task == nil || task.Identity == "" {
|
|
return nil
|
|
}
|
|
for _, id := range d.Identities {
|
|
if id.Name == task.Identity {
|
|
return id
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// forgeFor returns the forge client that owns task: the identity's own
|
|
// client when task.Identity names a configured identity, else the root
|
|
// d.Forge. This is the safety boundary that keeps one identity's forge
|
|
// token from being used against another identity's repos.
|
|
func (d *Daemon) forgeFor(task *store.Task) forge.Forge {
|
|
if id := d.identityFor(task); id != nil {
|
|
return id.Forge
|
|
}
|
|
return d.Forge
|
|
}
|
|
|
|
// treesFor returns the worktree manager that owns task, mirroring forgeFor.
|
|
func (d *Daemon) treesFor(task *store.Task) *worktree.Manager {
|
|
if id := d.identityFor(task); id != nil {
|
|
return id.Trees
|
|
}
|
|
return d.Trees
|
|
}
|
|
|
|
// repoFor resolves task's repo config from the owning identity's repo
|
|
// list when task.Identity is set, else the root Cfg.Repos.
|
|
func (d *Daemon) repoFor(t *store.Task) (config.Repo, bool) {
|
|
repos := d.Cfg.Get().Repos
|
|
if id := d.identityFor(t); id != nil {
|
|
repos = id.Repos
|
|
}
|
|
for _, r := range repos {
|
|
if r.Owner == t.Owner && r.Name == t.Repo {
|
|
return r, true
|
|
}
|
|
}
|
|
return config.Repo{}, false
|
|
}
|
|
|
|
// allowConcurrentForTask reports whether the task's owning repo has opted
|
|
// into concurrent task dispatch (config.Repo.AllowConcurrent). The repo is
|
|
// resolved through repoFor, which selects the identity-scoped repo list when
|
|
// task.Identity names a configured identity -- so multi-identity concurrency
|
|
// policy comes from the identity's own config, not the root daemon's.
|
|
// Unknown repos default to the safe, serialized behavior.
|
|
func (d *Daemon) allowConcurrentForTask(task *store.Task) bool {
|
|
repo, ok := d.repoFor(task)
|
|
return ok && repo.AllowConcurrent
|
|
}
|