fix(daemon): require full-task worker handoff

Autonomous tasks no longer execute workflow stages inside archied; missing worker capability parks before worktree preparation.

A transition to parked now atomically makes its detail the task park reason, so daemon-side failures remain visible in task APIs and the dashboard.
This commit is contained in:
sam committed 2026-08-24 14:56:57 +10:00
1 parent 1b7bc004b6
commit cc6b093820
6 files changed
+276 -134

No files matched your search

+23 -30
View File
@@ -104,9 +104,8 @@ type boot struct {
// startup. Embedded mode generates it; external mode resolves it once.
natsToken string
containerPool *container.Pool
storeBackend storage.Backend
sandboxRequired bool
containerPool *container.Pool
storeBackend storage.Backend
providers map[string]agentexec.Provider
llm *runtime.Runtime
@@ -428,10 +427,6 @@ func (b *boot) setupContainers(ctx context.Context) func() {
containerPool, storeBackend, closeDocker := startContainers(ctx, b.cfg, b.log)
b.containerPool = containerPool
b.storeBackend = storeBackend
// Sandboxing that was asked for and could not start must not silently
// become no sandboxing: the daemon stays up, but tasks park rather than
// running the agent loop on the host.
b.sandboxRequired = b.cfg.Containers.Enabled && containerPool == nil
return closeDocker
}
@@ -1006,29 +1001,27 @@ func (b *boot) registerMinimaxTool(cfg config.Config, log *slog.Logger) {
func (b *boot) buildDaemon() {
cfg, log := b.cfg, b.log
b.d = &daemon.Daemon{
Cfg: config.NewHolder(cfg),
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken},
Store: b.st,
Bus: b.bus,
Forge: b.forgeClient,
Trees: b.trees,
Agent: b.agentRunner,
Workflows: b.registry,
CapabilityHost: b.capabilityHost,
Storage: b.storeBackend,
Log: log,
CustomStages: wfeval.Discover,
Tasks: b.natsClient,
WorktreeGrants: b.worktreeGrants,
ContainerPool: b.containerPool,
SandboxRequired: b.sandboxRequired,
Memory: b.memManager,
Guardrails: b.guardrails,
ToolRegistry: b.toolReg,
Curators: b.curatorRegistry,
Identities: b.identityRunners,
TaskLogs: b.taskLogs,
AgentStatus: b.agentStatus,
Cfg: config.NewHolder(cfg),
ConnectedNATS: daemon.NATSEndpoint{URL: b.natsURL, Token: b.natsToken},
Store: b.st,
Bus: b.bus,
Forge: b.forgeClient,
Trees: b.trees,
Agent: b.agentRunner,
Workflows: b.registry,
CapabilityHost: b.capabilityHost,
Storage: b.storeBackend,
Log: log,
CustomStages: wfeval.Discover,
Tasks: b.natsClient,
WorktreeGrants: b.worktreeGrants,
ContainerPool: b.containerPool,
Guardrails: b.guardrails,
ToolRegistry: b.toolReg,
Curators: b.curatorRegistry,
Identities: b.identityRunners,
TaskLogs: b.taskLogs,
AgentStatus: b.agentStatus,
}
b.web.TaskStopper = b.d
// web and the daemon must share ONE Holder: a reload swaps d.Cfg and
+76
View File
@@ -0,0 +1,76 @@
package daemon
import (
"go/ast"
"go/parser"
"go/token"
"path/filepath"
"strconv"
"strings"
"testing"
)
const workflowImportPath = "github.com/samcharles93/archie-core/internal/domain/workflow"
// Autonomous workflow execution belongs exclusively to the task-scoped
// archie-agent worker. A local workflow.Run call in daemon silently restores
// the host-authority fallback this boundary exists to remove.
func TestDaemonNeverExecutesWorkflowLocally(t *testing.T) {
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
fileset := token.NewFileSet()
for _, path := range files {
if strings.HasSuffix(path, "_test.go") {
continue
}
file, parseErr := parser.ParseFile(fileset, path, nil, 0)
if parseErr != nil {
t.Errorf("parse %s: %v", path, parseErr)
continue
}
workflowNames := make(map[string]bool)
dotImport := false
for _, spec := range file.Imports {
importPath, unquoteErr := strconv.Unquote(spec.Path.Value)
if unquoteErr != nil || importPath != workflowImportPath {
continue
}
switch {
case spec.Name == nil:
workflowNames["workflow"] = true
case spec.Name.Name == ".":
dotImport = true
case spec.Name.Name != "_":
workflowNames[spec.Name.Name] = true
}
}
ast.Inspect(file, func(node ast.Node) bool {
call, ok := node.(*ast.CallExpr)
if !ok {
return true
}
if dotImport {
if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "Run" {
reportLocalWorkflowRun(t, fileset, call)
}
}
selector, ok := call.Fun.(*ast.SelectorExpr)
if !ok || selector.Sel.Name != "Run" {
return true
}
pkg, ok := selector.X.(*ast.Ident)
if ok && workflowNames[pkg.Name] {
reportLocalWorkflowRun(t, fileset, call)
}
return true
})
}
}
func reportLocalWorkflowRun(t *testing.T, fileset *token.FileSet, call *ast.CallExpr) {
t.Helper()
position := fileset.Position(call.Pos())
t.Errorf("%s calls workflow.Run; autonomous workflows must use the full-task worker handoff", position)
}
+41 -97
View File
@@ -19,14 +19,12 @@ import (
"github.com/samcharles93/archie-core/internal/container"
"github.com/samcharles93/archie-core/internal/domain/curator"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/domain/workflow/skillbuild"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/eventbus"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
"github.com/samcharles93/archie-core/internal/logging"
"github.com/samcharles93/archie-core/internal/memory"
"github.com/samcharles93/archie-core/internal/plugin"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/store"
@@ -95,14 +93,6 @@ type Daemon struct {
// TaskRunRetryBackoff is the delay between retry attempts within
// TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff.
TaskRunRetryBackoff time.Duration
// SandboxRequired reports that [containers] asked for sandboxing that
// could not be brought up. It exists because ContainerPool == nil means
// two very different things: "sandboxing was never configured, run
// in-process" and "sandboxing was configured and failed". Treating them
// alike silently dropped the isolation boundary and ran agent loops on
// the host with the daemon's own authority.
SandboxRequired bool
// AgentStatus records the version/install-type the most recently
// completed archie-agent task response reported about itself, for
// releaseupdate.Report.Verify to check an update claim against. Nil
@@ -111,11 +101,12 @@ type Daemon struct {
// would read as a checked "unknown" rather than "never wired".
AgentStatus *AgentStatus
// ContainerPool manages Docker container lifecycle. Nil when [containers]
// is not configured. When non-nil, every task gets a fresh container.
// ContainerPool manages Docker container lifecycle. Nil means autonomous
// execution is unavailable and process parks tasks; every runnable task gets
// a fresh container.
ContainerPool *container.Pool
// Storage is the pluggable storage backend for container mounts.
// When nil (no containers configured), mount setup is skipped.
// A runnable task requires it; acquireTaskContainer parks on nil.
Storage storage.Backend
// CapabilityHost owns validated plugin manifests and cross-family
// lifecycle. Typed capability registries remain in their domain packages;
@@ -135,12 +126,6 @@ type Daemon struct {
// using the single-identity Forge/Trees/Cfg.Repos path.
Identities []*IdentityRunner
// Memory is the pluggable memory manager, wired by the composition root
// (cmd/archied). When non-nil, the system prompt includes memory context
// and memory tool calls are routed through the manager. Nil means memory
// is disabled (backward compatible).
Memory *memory.Manager
// Guardrails is the tool-call guardrail engine, wired by the composition
// root. When non-nil, tool successes and failures are recorded and
// warnings/hard-stops are issued per the configured thresholds. Nil
@@ -166,15 +151,6 @@ type Daemon struct {
running runningTasks
}
// memoryPrompt returns the memory manager's system prompt block, or an empty
// string when memory is disabled. Safe to call from any goroutine.
func (d *Daemon) memoryPrompt() string {
if d.Memory == nil {
return ""
}
return d.Memory.SystemPromptBlock()
}
// IdentityRunner bundles identity-specific state for a single agent
// identity running within a multi-identity daemon. Each identity gets its
// own forge client, worktree manager, repo list, and config -- but shares
@@ -839,35 +815,42 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
defer finished()
defer d.openTaskLog(task)()
fg := d.forgeFor(task)
trees := d.treesFor(task)
executionCfg, executionAgent := d.executionFor(task)
repo, ok := d.repoFor(task)
if !ok {
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, "repo no longer in config")
return
}
// Per-worktree registry augmentation: if the worktree already
// exists (e.g. retry, waiting_human → implement handoff), scan
// it for .agents/skills/ that declare workflows. Worktree skills
// override the startup registry for this task only.
workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
registry := d.Workflows
if _, err := os.Stat(workDir); err == nil {
aug, err := skillbuild.AugmentRegistry(workDir, d.Workflows)
if err != nil {
d.Log.Error("worktree registry augmentation failed -- using startup registry", "dir", workDir, "err", err)
d.emit(events.Event{
Kind: "registry_augment_failed", TaskID: task.ID,
Repo: task.Owner + "/" + task.Repo, Issue: task.IssueNumber,
Detail: err.Error(),
})
} else {
registry = aug
if d.ContainerPool == nil {
const reason = "managed agent container pool is unavailable; refusing to run this task on the host"
d.Log.Error("task parked: managed worker unavailable", "task", task.ID,
"hint", "enable containers and make the archie-agent image available, then retry the task")
if err := d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason); err != nil {
d.Log.Warn("managed worker park transition failed", "task", task.ID, "err", err)
return
}
d.emit(events.Event{
Kind: events.KindParked, TaskID: task.ID,
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
})
return
}
wf := workflow.Route(task, registry)
d.Log.Info("processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "workflow", wf.Name, "attempt", task.Attempt)
if d.Tasks == nil {
const reason = "agent task transport is unavailable; refusing to run this task on the host"
d.Log.Error("task parked: agent task transport unavailable", "task", task.ID)
if err := d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason); err != nil {
d.Log.Warn("agent task transport park transition failed", "task", task.ID, "err", err)
return
}
d.emit(events.Event{
Kind: events.KindParked, TaskID: task.ID,
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
})
return
}
workDir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
d.Log.Info("processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
// Clone the worktree before Docker mount setup. The container
// binds /data/worktree to workDir on the host -- the directory
@@ -890,60 +873,21 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
d.Log.Warn("task branch not persisted", "task", task.ID, "err", err)
}
// Refuse to run unsandboxed when sandboxing was asked for. Parking is
// recoverable -- fix the image, retry -- whereas running the agent loop
// in-process would hand it the host filesystem and the daemon's own
// authority, which is the one thing [containers] exists to prevent. The
// daemon itself stays up: chat and the dashboard are unaffected.
if d.SandboxRequired && d.ContainerPool == nil {
const reason = "containers are enabled but the pool is unavailable; " +
"refusing to run this task unsandboxed on the host"
d.Log.Error("task parked: sandbox unavailable", "task", task.ID,
"hint", "run `docker compose pull agent` (or `build agent`), then retry the task")
_ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason)
d.emit(events.Event{
Kind: events.KindParked, TaskID: task.ID,
Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason,
})
ctr, ok := d.acquireTaskContainer(ctx, task, repo, workDir)
if !ok {
return
}
defer d.ContainerPool.Release(ctx, ctr)
// Acquire a container for the task when Docker sandboxing is enabled.
if d.ContainerPool != nil {
ctr, ok := d.acquireTaskContainer(ctx, task, repo, workDir)
if !ok {
return
}
defer d.ContainerPool.Release(ctx, ctr)
}
// Sandboxed path: hand the whole task to archie-agent in one NATS round
// trip instead of running the stage loop here. archie-agent proxies
// Store/Forge/worktree-push calls back to archied over storerpc/
// Hand the whole task to archie-agent in one NATS round trip. archie-agent
// proxies Store/Forge/worktree-push calls back to archied over storerpc/
// forgerpc/worktreerpc -- by the time runViaAgent returns, the task's
// terminal state already landed via those calls. The task's identity
// travels in the taskrun request; the agent scopes its forgerpc and
// worktreerpc clients to that identity, and the daemon registered one
// server pair per identity (plus the root pair), so a container-mode
// task is always served by its own forge client and worktree manager.
if d.ContainerPool != nil {
d.runViaAgent(ctx, task, repo)
} else {
workflow.Run(ctx, wf, &workflow.TaskContext{
Task: task,
Repo: repo,
Cfg: executionCfg,
Forge: fg,
Store: d.Store,
Trees: trees,
SystemPrompt: d.memoryPrompt,
Guardrails: d.Guardrails,
Agent: executionAgent,
Bus: d.Bus,
Log: d.Log,
CustomStages: d.CustomStages,
})
}
d.runViaAgent(ctx, task, repo)
// Teardown storage after workflow completes. The Docker backend is a
// no-op; future backends (temp volumes, NFS leases) use this hook.
@@ -959,9 +903,9 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) {
}
// openTaskLog opens task's log sink for the lifetime of one process() call
// and returns the function to close it, covering both dispatch paths below
// (sandboxed container and in-process) uniformly regardless of which one
// runs. A failure to open is not fatal to the task -- it means this
// and returns the function to close it. That lifetime covers preparation,
// the full-task container handoff, and teardown. A failure to open is not
// fatal to the task -- it means this
// attempt's own output won't be recoverable if something goes wrong, which
// is worse than the pre-existing state, not equal to it, so the run
// proceeds rather than parking over a logging problem.
+98
View File
@@ -7,6 +7,7 @@ import (
"log/slog"
"os"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
@@ -17,7 +18,9 @@ import (
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/container"
"github.com/samcharles93/archie-core/internal/domain/workintake"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/forge"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
arnats "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
@@ -93,6 +96,90 @@ func TestOpenTaskLogOpensAndClosesTheRightSink(t *testing.T) {
}
}
func TestProcessParksWhenManagedWorkerCapabilityIsUnavailable(t *testing.T) {
tests := []struct {
name string
pool *container.Pool
wantReason string
}{
{
name: "container pool unavailable",
wantReason: "managed agent container pool is unavailable; refusing to run this task on the host",
},
{
name: "task transport unavailable",
pool: &container.Pool{},
wantReason: "agent task transport is unavailable; refusing to run this task on the host",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
d, st, _ := testDaemon(t, 3, 0)
d.Log = slog.New(slog.DiscardHandler)
d.Trees = nil // both guards must run before any worktree operation
d.ContainerPool = tt.pool
d.Cfg.Set(config.Config{
Repos: []config.Repo{{Owner: "acme", Name: "widget"}},
})
ctx := t.Context()
if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil {
t.Fatal(err)
}
task, err := st.ClaimNext(ctx)
if err != nil {
t.Fatal(err)
}
if task == nil {
t.Fatal("ClaimNext = nil task")
}
d.process(ctx, task)
got, err := st.TaskByID(ctx, task.ID)
if err != nil {
t.Fatal(err)
}
if got.Status != store.StatusParked {
t.Fatalf("task status = %q, want parked", got.Status)
}
if got.ParkReason != tt.wantReason {
t.Errorf("park reason = %q, want %q", got.ParkReason, tt.wantReason)
}
})
}
}
func TestProcessDoesNotEmitParkedWhenCapabilityTransitionLosesRace(t *testing.T) {
d, st, _ := testDaemon(t, 3, 0)
d.Log = slog.New(slog.DiscardHandler)
d.Trees = nil
d.Cfg.Set(config.Config{Repos: []config.Repo{{Owner: "acme", Name: "widget"}}})
d.Bus = events.NewBus()
sub := d.Bus.Subscribe(1)
t.Cleanup(sub.Close)
ctx := t.Context()
if _, err := st.EnqueueIssue(ctx, "acme", "widget", 42, "task", "", "", ""); err != nil {
t.Fatal(err)
}
task, err := st.TaskByIssue(ctx, "acme", "widget", 42)
if err != nil || task == nil {
t.Fatalf("TaskByIssue = (%+v, %v)", task, err)
}
// The task is still queued, simulating another actor winning the guarded
// running-state transition before this process reports its capability
// failure.
d.process(ctx, task)
select {
case event := <-sub.C:
t.Fatalf("event = %+v, want no false parked event after stale transition", event)
default:
}
}
func TestTaskDispatcherEnforcesMaxConcurrency(t *testing.T) {
t.Parallel()
@@ -464,6 +551,9 @@ func TestRunViaAgentParksOnRequestFailure(t *testing.T) {
if got.Status != store.StatusParked {
t.Fatalf("status = %q, want %q", got.Status, store.StatusParked)
}
if !strings.Contains(got.ParkReason, "taskrun request failed") {
t.Errorf("ParkReason = %q, want taskrun request failure", got.ParkReason)
}
}
// TestRunViaAgentRetriesUntilResponderAppears is a regression test for the
@@ -580,6 +670,9 @@ func TestRunViaAgentParksOnRunError(t *testing.T) {
if got.Status != store.StatusParked {
t.Fatalf("status = %q, want %q", got.Status, store.StatusParked)
}
if got.ParkReason != "taskrun run failed: registry build failed" {
t.Errorf("ParkReason = %q, want worker run failure", got.ParkReason)
}
}
func TestRunViaAgentSendsExpectedRequest(t *testing.T) {
@@ -598,7 +691,9 @@ func TestRunViaAgentSendsExpectedRequest(t *testing.T) {
}
received := make(chan taskrun.Request, 1)
var requestCount atomic.Int32
sub, err := mustCoreConn(t, busClient).Subscribe(agentnats.SubjectForTask(task.ID), func(msg *natsio.Msg) {
requestCount.Add(1)
var req taskrun.Request
_ = json.Unmarshal(msg.Data, &req)
received <- req
@@ -630,6 +725,9 @@ func TestRunViaAgentSendsExpectedRequest(t *testing.T) {
case <-time.After(time.Second):
t.Fatal("archied did not publish a taskrun request")
}
if got := requestCount.Load(); got != 1 {
t.Errorf("taskrun request count = %d, want exactly 1", got)
}
// Success path: runViaAgent must not park a task that archie-agent
// already reported as complete -- the daemon just logs it.
+13 -7
View File
@@ -349,11 +349,12 @@ func (s *Store) ClaimByIssue(ctx context.Context, owner, repo string, number int
// the task's current status does not match the expected from status.
var ErrStaleTransition = errors.New("store: stale transition: task status does not match expected from status")
// Transition moves a task to a new status, recording the change.
// The from status acts as a guard: the UPDATE only affects rows whose
// current status matches from. If no row matches, ErrStaleTransition is
// returned and no audit row is written. Both statements execute in a
// single transaction.
// Transition moves a task to a new status and records the audit detail. The
// from status guards the update; a mismatch returns ErrStaleTransition without
// writing an audit row. Transitioning to parked also stores detail as
// ParkReason in the same transaction. Detail used to exist only in the
// timeline, which left daemon-side parks looking reasonless in task APIs and
// the dashboard.
func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail string) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
@@ -361,8 +362,13 @@ func (s *Store) Transition(ctx context.Context, taskID int64, from, to, detail s
}
defer func() { _ = tx.Rollback() }()
res, err := tx.ExecContext(ctx,
`UPDATE tasks SET status=?, updated_at=datetime('now') WHERE id=? AND status=?`, to, taskID, from)
res, err := tx.ExecContext(ctx, `
UPDATE tasks
SET status=?,
park_reason=CASE WHEN ?=? THEN ? ELSE park_reason END,
updated_at=datetime('now')
WHERE id=? AND status=?`,
to, to, StatusParked, clip(detail, 4000), taskID, from)
if err != nil {
return err
}
+25
View File
@@ -583,6 +583,31 @@ func TestTransitionPreventsDoubleTransition(t *testing.T) {
}
}
func TestTransitionToParkedPersistsReasonOnTask(t *testing.T) {
s := openTest(t)
ctx := t.Context()
if _, err := s.EnqueueIssue(ctx, "acme", "widget", 1, "task", "", "", ""); err != nil {
t.Fatal(err)
}
task, err := s.ClaimNext(ctx)
if err != nil || task == nil {
t.Fatalf("ClaimNext = (%+v, %v)", task, err)
}
const reason = "managed worker unavailable"
if err := s.Transition(ctx, task.ID, StatusRunning, StatusParked, reason); err != nil {
t.Fatal(err)
}
got, err := s.TaskByID(ctx, task.ID)
if err != nil {
t.Fatal(err)
}
if got.ParkReason != reason {
t.Errorf("ParkReason = %q, want %q", got.ParkReason, reason)
}
}
// TestRequeueRejectsStaleFrom proves that Requeue rejects a call whose
// fromStatus does not match the task's current status.
func TestRequeueRejectsStaleFrom(t *testing.T) {