From 5303ef65db461729f8e61743f29c733c9d3d415b Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 24 Aug 2026 13:49:30 +1000 Subject: [PATCH] fix(daemon): require full-task worker handoff Autonomous tasks no longer execute workflow stages inside archied; missing worker capability parks before worktree preparation. A transition to parked now atomically makes its detail the task park reason, so daemon-side failures remain visible in task APIs and the dashboard. --- cmd/archied/bootstrap.go | 53 +++++----- internal/daemon/architecture_test.go | 76 +++++++++++++++ internal/daemon/daemon.go | 138 ++++++++------------------- internal/daemon/daemon_test.go | 98 +++++++++++++++++++ internal/store/store.go | 20 ++-- internal/store/store_test.go | 25 +++++ 6 files changed, 276 insertions(+), 134 deletions(-) create mode 100644 internal/daemon/architecture_test.go diff --git a/cmd/archied/bootstrap.go b/cmd/archied/bootstrap.go index 2e9ac0bae..e39bcac20 100644 --- a/cmd/archied/bootstrap.go +++ b/cmd/archied/bootstrap.go @@ -104,9 +104,8 @@ type boot struct { // startup. Embedded mode generates it; external mode resolves it once. natsToken string - containerPool *container.Pool - storeBackend storage.Backend - sandboxRequired bool + containerPool *container.Pool + storeBackend storage.Backend providers map[string]agentexec.Provider llm *runtime.Runtime @@ -428,10 +427,6 @@ func (b *boot) setupContainers(ctx context.Context) func() { containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfg, b.log) b.containerPool = containerPool b.storeBackend = storeBackend - // Sandboxing that was asked for and could not start must not silently - // become no sandboxing: the daemon stays up, but tasks park rather than - // running the agent loop on the host. - b.sandboxRequired = b.cfg.Containers.Enabled && containerPool == nil return closeDocker } @@ -1006,29 +1001,27 @@ func (b *boot) registerMinimaxTool(cfg config.Config, log *slog.Logger) { func (b *boot) buildDaemon() { cfg, log := b.cfg, b.log b.d = &daemon.Daemon{ - Cfg: config.NewHolder(cfg), - ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken}, - Store: b.st, - Bus: b.bus, - Forge: b.forgeClient, - Trees: b.trees, - Agent: b.agentRunner, - Workflows: b.registry, - CapabilityHost: b.capabilityHost, - Storage: b.storeBackend, - Log: log, - CustomStages: wfeval.Discover, - Tasks: b.natsClient, - WorktreeGrants: b.worktreeGrants, - ContainerPool: b.containerPool, - SandboxRequired: b.sandboxRequired, - Memory: b.memManager, - Guardrails: b.guardrails, - ToolRegistry: b.toolReg, - Curators: b.curatorRegistry, - Identities: b.identityRunners, - TaskLogs: b.taskLogs, - AgentStatus: b.agentStatus, + Cfg: config.NewHolder(cfg), + ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken}, + Store: b.st, + Bus: b.bus, + Forge: b.forgeClient, + Trees: b.trees, + Agent: b.agentRunner, + Workflows: b.registry, + CapabilityHost: b.capabilityHost, + Storage: b.storeBackend, + Log: log, + CustomStages: wfeval.Discover, + Tasks: b.natsClient, + WorktreeGrants: b.worktreeGrants, + ContainerPool: b.containerPool, + Guardrails: b.guardrails, + ToolRegistry: b.toolReg, + Curators: b.curatorRegistry, + Identities: b.identityRunners, + TaskLogs: b.taskLogs, + AgentStatus: b.agentStatus, } b.web.TaskStopper = b.d // web and the daemon must share ONE Holder: a reload swaps d.Cfg and diff --git a/internal/daemon/architecture_test.go b/internal/daemon/architecture_test.go new file mode 100644 index 000000000..e11086c8c --- /dev/null +++ b/internal/daemon/architecture_test.go @@ -0,0 +1,76 @@ +package daemon + +import ( + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const workflowImportPath = "github.com/samcharles93/archie-core/internal/domain/workflow" + +// Autonomous workflow execution belongs exclusively to the task-scoped +// archie-agent worker. A local workflow.Run call in daemon silently restores +// the host-authority fallback this boundary exists to remove. +func TestDaemonNeverExecutesWorkflowLocally(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + fileset := token.NewFileSet() + for _, path := range files { + if strings.HasSuffix(path, "_test.go") { + continue + } + file, parseErr := parser.ParseFile(fileset, path, nil, 0) + if parseErr != nil { + t.Errorf("parse %s: %v", path, parseErr) + continue + } + workflowNames := make(map[string]bool) + dotImport := false + for _, spec := range file.Imports { + importPath, unquoteErr := strconv.Unquote(spec.Path.Value) + if unquoteErr != nil || importPath != workflowImportPath { + continue + } + switch { + case spec.Name == nil: + workflowNames["workflow"] = true + case spec.Name.Name == ".": + dotImport = true + case spec.Name.Name != "_": + workflowNames[spec.Name.Name] = true + } + } + ast.Inspect(file, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if dotImport { + if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "Run" { + reportLocalWorkflowRun(t, fileset, call) + } + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "Run" { + return true + } + pkg, ok := selector.X.(*ast.Ident) + if ok && workflowNames[pkg.Name] { + reportLocalWorkflowRun(t, fileset, call) + } + return true + }) + } +} + +func reportLocalWorkflowRun(t *testing.T, fileset *token.FileSet, call *ast.CallExpr) { + t.Helper() + position := fileset.Position(call.Pos()) + t.Errorf("%s calls workflow.Run; autonomous workflows must use the full-task worker handoff", position) +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 3c878c6e9..d0d7690f7 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -19,14 +19,12 @@ import ( "github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/domain/curator" "github.com/samcharles93/archie-core/internal/domain/workflow" - "github.com/samcharles93/archie-core/internal/domain/workflow/skillbuild" "github.com/samcharles93/archie-core/internal/domain/workintake" "github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/forge" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" "github.com/samcharles93/archie-core/internal/logging" - "github.com/samcharles93/archie-core/internal/memory" "github.com/samcharles93/archie-core/internal/plugin" "github.com/samcharles93/archie-core/internal/storage" "github.com/samcharles93/archie-core/internal/store" @@ -95,14 +93,6 @@ type Daemon struct { // TaskRunRetryBackoff is the delay between retry attempts within // TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff. TaskRunRetryBackoff time.Duration - // SandboxRequired reports that [containers] asked for sandboxing that - // could not be brought up. It exists because ContainerPool == nil means - // two very different things: "sandboxing was never configured, run - // in-process" and "sandboxing was configured and failed". Treating them - // alike silently dropped the isolation boundary and ran agent loops on - // the host with the daemon's own authority. - SandboxRequired bool - // AgentStatus records the version/install-type the most recently // completed archie-agent task response reported about itself, for // releaseupdate.Report.Verify to check an update claim against. Nil @@ -111,11 +101,12 @@ type Daemon struct { // would read as a checked "unknown" rather than "never wired". AgentStatus *AgentStatus - // ContainerPool manages Docker container lifecycle. Nil when [containers] - // is not configured. When non-nil, every task gets a fresh container. + // ContainerPool manages Docker container lifecycle. Nil means autonomous + // execution is unavailable and process parks tasks; every runnable task gets + // a fresh container. ContainerPool *container.Pool // Storage is the pluggable storage backend for container mounts. - // When nil (no containers configured), mount setup is skipped. + // A runnable task requires it; acquireTaskContainer parks on nil. Storage storage.Backend // CapabilityHost owns validated plugin manifests and cross-family // lifecycle. Typed capability registries remain in their domain packages; @@ -135,12 +126,6 @@ type Daemon struct { // using the single-identity Forge/Trees/Cfg.Repos path. Identities []*IdentityRunner - // Memory is the pluggable memory manager, wired by the composition root - // (cmd/archied). When non-nil, the system prompt includes memory context - // and memory tool calls are routed through the manager. Nil means memory - // is disabled (backward compatible). - Memory *memory.Manager - // Guardrails is the tool-call guardrail engine, wired by the composition // root. When non-nil, tool successes and failures are recorded and // warnings/hard-stops are issued per the configured thresholds. Nil @@ -166,15 +151,6 @@ type Daemon struct { running runningTasks } -// memoryPrompt returns the memory manager's system prompt block, or an empty -// string when memory is disabled. Safe to call from any goroutine. -func (d *Daemon) memoryPrompt() string { - if d.Memory == nil { - return "" - } - return d.Memory.SystemPromptBlock() -} - // IdentityRunner bundles identity-specific state for a single agent // identity running within a multi-identity daemon. Each identity gets its // own forge client, worktree manager, repo list, and config -- but shares @@ -839,35 +815,42 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { defer finished() defer d.openTaskLog(task)() - fg := d.forgeFor(task) trees := d.treesFor(task) - executionCfg, executionAgent := d.executionFor(task) repo, ok := d.repoFor(task) if !ok { _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "repo no longer in config") return } - // Per-worktree registry augmentation: if the worktree already - // exists (e.g. retry, waiting_human → implement handoff), scan - // it for .agents/skills/ that declare workflows. Worktree skills - // override the startup registry for this task only. - workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber) - registry := d.Workflows - if _, err := os.Stat(workDir); err == nil { - aug, err := skillbuild.AugmentRegistry(workDir, d.Workflows) - if err != nil { - d.Log.Error("worktree registry augmentation failed -- using startup registry", "dir", workDir, "err", err) - d.emit(events.Event{ - Kind: "registry_augment_failed", TaskID: task.ID, - Repo: task.Owner + "/" + task.Repo, Issue: task.IssueNumber, - Detail: err.Error(), - }) - } else { - registry = aug + if d.ContainerPool == nil { + const reason = "managed agent container pool is unavailable; refusing to run this task on the host" + d.Log.Error("task parked: managed worker unavailable", "task", task.ID, + "hint", "enable containers and make the archie-agent image available, then retry the task") + if err := d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason); err != nil { + d.Log.Warn("managed worker park transition failed", "task", task.ID, "err", err) + return } + d.emit(events.Event{ + Kind: events.KindParked, TaskID: task.ID, + Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason, + }) + return } - wf := workflow.Route(task, registry) - d.Log.Info("processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "workflow", wf.Name, "attempt", task.Attempt) + if d.Tasks == nil { + const reason = "agent task transport is unavailable; refusing to run this task on the host" + d.Log.Error("task parked: agent task transport unavailable", "task", task.ID) + if err := d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason); err != nil { + d.Log.Warn("agent task transport park transition failed", "task", task.ID, "err", err) + return + } + d.emit(events.Event{ + Kind: events.KindParked, TaskID: task.ID, + Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason, + }) + return + } + + workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber) + d.Log.Info("processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt) // Clone the worktree before Docker mount setup. The container // binds /data/worktree to workDir on the host -- the directory @@ -890,60 +873,21 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { d.Log.Warn("task branch not persisted", "task", task.ID, "err", err) } - // Refuse to run unsandboxed when sandboxing was asked for. Parking is - // recoverable -- fix the image, retry -- whereas running the agent loop - // in-process would hand it the host filesystem and the daemon's own - // authority, which is the one thing [containers] exists to prevent. The - // daemon itself stays up: chat and the dashboard are unaffected. - if d.SandboxRequired && d.ContainerPool == nil { - const reason = "containers are enabled but the pool is unavailable; " + - "refusing to run this task unsandboxed on the host" - d.Log.Error("task parked: sandbox unavailable", "task", task.ID, - "hint", "run `docker compose pull agent` (or `build agent`), then retry the task") - _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason) - d.emit(events.Event{ - Kind: events.KindParked, TaskID: task.ID, - Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason, - }) + ctr, ok := d.acquireTaskContainer(ctx, task, repo, workDir) + if !ok { return } + defer d.ContainerPool.Release(ctx, ctr) - // Acquire a container for the task when Docker sandboxing is enabled. - if d.ContainerPool != nil { - ctr, ok := d.acquireTaskContainer(ctx, task, repo, workDir) - if !ok { - return - } - defer d.ContainerPool.Release(ctx, ctr) - } - - // Sandboxed path: hand the whole task to archie-agent in one NATS round - // trip instead of running the stage loop here. archie-agent proxies - // Store/Forge/worktree-push calls back to archied over storerpc/ + // Hand the whole task to archie-agent in one NATS round trip. archie-agent + // proxies Store/Forge/worktree-push calls back to archied over storerpc/ // forgerpc/worktreerpc -- by the time runViaAgent returns, the task's // terminal state already landed via those calls. The task's identity // travels in the taskrun request; the agent scopes its forgerpc and // worktreerpc clients to that identity, and the daemon registered one // server pair per identity (plus the root pair), so a container-mode // task is always served by its own forge client and worktree manager. - if d.ContainerPool != nil { - d.runViaAgent(ctx, task, repo) - } else { - workflow.Run(ctx, wf, &workflow.TaskContext{ - Task: task, - Repo: repo, - Cfg: executionCfg, - Forge: fg, - Store: d.Store, - Trees: trees, - SystemPrompt: d.memoryPrompt, - Guardrails: d.Guardrails, - Agent: executionAgent, - Bus: d.Bus, - Log: d.Log, - CustomStages: d.CustomStages, - }) - } + d.runViaAgent(ctx, task, repo) // Teardown storage after workflow completes. The Docker backend is a // no-op; future backends (temp volumes, NFS leases) use this hook. @@ -959,9 +903,9 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { } // openTaskLog opens task's log sink for the lifetime of one process() call -// and returns the function to close it, covering both dispatch paths below -// (sandboxed container and in-process) uniformly regardless of which one -// runs. A failure to open is not fatal to the task -- it means this +// and returns the function to close it. That lifetime covers preparation, +// the full-task container handoff, and teardown. A failure to open is not +// fatal to the task -- it means this // attempt's own output won't be recoverable if something goes wrong, which // is worse than the pre-existing state, not equal to it, so the run // proceeds rather than parking over a logging problem. diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index c78d9dbe2..d2736ae3b 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -7,6 +7,7 @@ import ( "log/slog" "os" "slices" + "strings" "sync/atomic" "testing" "time" @@ -17,7 +18,9 @@ import ( "github.com/samcharles93/archie-core/internal/agentexec" "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/domain/workintake" + "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/forge" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" @@ -93,6 +96,90 @@ func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) { } } +func TestProcessParksWhenManagedWorkerCapabilityIsUnavailable(t *testing.T) { + tests := []struct { + name string + pool *container.Pool + wantReason string + }{ + { + name: "container pool unavailable", + wantReason: "managed agent container pool is unavailable; refusing to run this task on the host", + }, + { + name: "task transport unavailable", + pool: &container.Pool{}, + wantReason: "agent task transport is unavailable; refusing to run this task on the host", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d, st, _ := testDaemon(t, 3, 0) + d.Log = slog.New(slog.DiscardHandler) + d.Trees = nil // both guards must run before any worktree operation + d.ContainerPool = tt.pool + d.Cfg.Set(config.Config{ + Repos: []config.Repo{{Owner: "acme", Name: "widget"}}, + }) + + ctx := t.Context() + if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { + t.Fatal(err) + } + task, err := st.ClaimNext(ctx) + if err != nil { + t.Fatal(err) + } + if task == nil { + t.Fatal("ClaimNext = nil task") + } + + d.process(ctx, task) + + got, err := st.TaskByID(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + if got.Status != store.StatusParked { + t.Fatalf("task status = %q, want parked", got.Status) + } + if got.ParkReason != tt.wantReason { + t.Errorf("park reason = %q, want %q", got.ParkReason, tt.wantReason) + } + }) + } +} + +func TestProcessDoesNotEmitParkedWhenCapabilityTransitionLosesRace(t *testing.T) { + d, st, _ := testDaemon(t, 3, 0) + d.Log = slog.New(slog.DiscardHandler) + d.Trees = nil + d.Cfg.Set(config.Config{Repos: []config.Repo{{Owner: "acme", Name: "widget"}}}) + d.Bus = events.NewBus() + sub := d.Bus.Subscribe(1) + t.Cleanup(sub.Close) + + ctx := t.Context() + if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil { + t.Fatal(err) + } + task, err := st.TaskByIssue(ctx, "acme", "widget", 42) + if err != nil || task == nil { + t.Fatalf("TaskByIssue = (%+v, %v)", task, err) + } + // The task is still queued, simulating another actor winning the guarded + // running-state transition before this process reports its capability + // failure. + d.process(ctx, task) + + select { + case event := <-sub.C: + t.Fatalf("event = %+v, want no false parked event after stale transition", event) + default: + } +} + func TestTaskDispatcherEnforcesMaxConcurrency(t *testing.T) { t.Parallel() @@ -464,6 +551,9 @@ func TestRunViaAgentParksOnRequestFailure(t *testing.T) { if got.Status != store.StatusParked { t.Fatalf("status = %q, want %q", got.Status, store.StatusParked) } + if !strings.Contains(got.ParkReason, "taskrun request failed") { + t.Errorf("ParkReason = %q, want taskrun request failure", got.ParkReason) + } } // TestRunViaAgentRetriesUntilResponderAppears is a regression test for the @@ -580,6 +670,9 @@ func TestRunViaAgentParksOnRunError(t *testing.T) { if got.Status != store.StatusParked { t.Fatalf("status = %q, want %q", got.Status, store.StatusParked) } + if got.ParkReason != "taskrun run failed: registry build failed" { + t.Errorf("ParkReason = %q, want worker run failure", got.ParkReason) + } } func TestRunViaAgentSendsExpectedRequest(t *testing.T) { @@ -598,7 +691,9 @@ func TestRunViaAgentSendsExpectedRequest(t *testing.T) { } received := make(chan taskrun.Request, 1) + var requestCount atomic.Int32 sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) { + requestCount.Add(1) var req taskrun.Request _ = json.Unmarshal(msg.Data, &req) received <- req @@ -630,6 +725,9 @@ func TestRunViaAgentSendsExpectedRequest(t *testing.T) { case <-time.After(time.Second): t.Fatal("archied did not publish a taskrun request") } + if got := requestCount.Load(); got != 1 { + t.Errorf("taskrun request count = %d, want exactly 1", got) + } // Success path: runViaAgent must not park a task that archie-agent // already reported as complete -- the daemon just logs it. diff --git a/internal/store/store.go b/internal/store/store.go index 6e25930d4..aace726c1 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -349,11 +349,12 @@ func (s *Store) ClaimByIssue(ctx context.Context, owner, repo string, number int // the task's current status does not match the expected from status. var ErrStaleTransition = errors.New("store: stale transition: task status does not match expected from status") -// Transition moves a task to a new status, recording the change. -// The from status acts as a guard: the UPDATE only affects rows whose -// current status matches from. If no row matches, ErrStaleTransition is -// returned and no audit row is written. Both statements execute in a -// single transaction. +// Transition moves a task to a new status and records the audit detail. The +// from status guards the update; a mismatch returns ErrStaleTransition without +// writing an audit row. Transitioning to parked also stores detail as +// ParkReason in the same transaction. Detail used to exist only in the +// timeline, which left daemon-side parks looking reasonless in task APIs and +// the dashboard. func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail string) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { @@ -361,8 +362,13 @@ func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail s } defer func() { _ = tx.Rollback() }() - res, err := tx.ExecContext(ctx, - `UPDATE tasks SET status=?, updated_at=datetime('now') WHERE id=? AND status=?`, to, taskID, from) + res, err := tx.ExecContext(ctx, ` + UPDATE tasks + SET status=?, + park_reason=CASE WHEN ?=? THEN ? ELSE park_reason END, + updated_at=datetime('now') + WHERE id=? AND status=?`, + to, to, StatusParked, clip(detail, 4000), taskID, from) if err != nil { return err } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 73b3f7a85..9d211e133 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -583,6 +583,31 @@ func TestTransitionPreventsDoubleTransition(t *testing.T) { } } +func TestTransitionToParkedPersistsReasonOnTask(t *testing.T) { + s := openTest(t) + ctx := t.Context() + + if _, err := s.EnqueueIssue(ctx, "acme", "widget", 1, "task", "", "", ""); err != nil { + t.Fatal(err) + } + task, err := s.ClaimNext(ctx) + if err != nil || task == nil { + t.Fatalf("ClaimNext = (%+v, %v)", task, err) + } + const reason = "managed worker unavailable" + if err := s.Transition(ctx, task.ID, StatusRunning, StatusParked, reason); err != nil { + t.Fatal(err) + } + + got, err := s.TaskByID(ctx, task.ID) + if err != nil { + t.Fatal(err) + } + if got.ParkReason != reason { + t.Errorf("ParkReason = %q, want %q", got.ParkReason, reason) + } +} + // TestRequeueRejectsStaleFrom proves that Requeue rejects a call whose // fromStatus does not match the task's current status. func TestRequeueRejectsStaleFrom(t *testing.T) {