fix(agentworker): supply the guardrail engine to every task run
TaskContext.Guardrails was never assigned: the engine was built in archied's composition root and held on Daemon.Guardrails, which no code read after workflow execution moved to the full-task worker handoff. archie-agent is a separate process, so a live engine cannot cross that handoff. Build the engine at the agentworker composition root (the process that runs agent stages), thread it through the task dependencies, and set it on TaskContext; drop the now-dead archied-side construction. Refs: archie-core-jd5v
This commit is contained in:
1 parent
4785ec1fc6
commit
88fb3c4f03
8 files changed
+132
-43
No files matched your search
@@ -0,0 +1,93 @@
|
||||
package agentworker
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
)
|
||||
|
||||
// guardrailRunYAML is a repository-none workflow with one agent stage: a task
|
||||
// run reaches AgentStage.handleResult once, and the engine that recorded that
|
||||
// outcome is observable by its call count and by the decision a low threshold
|
||||
// earns.
|
||||
const guardrailRunYAML = "id: guardrail\nrepository: none\nsteps:\n" +
|
||||
" - type: agent.run\n settings:\n mission: first\n"
|
||||
|
||||
// TestProductionWorkerDependenciesBuildGuardrails pins the producer: the
|
||||
// process that runs agent stages is archie-agent, so its composition root must
|
||||
// build the guardrail engine every TaskContext records against. A daemon-built
|
||||
// engine can never reach this process -- it is a separate binary -- so before
|
||||
// this wiring TaskContext.Guardrails was nil on every production run and no
|
||||
// decision was ever recorded.
|
||||
func TestProductionWorkerDependenciesBuildGuardrails(t *testing.T) {
|
||||
deps, err := productionWorkerDependencies()
|
||||
if err != nil {
|
||||
t.Fatalf("productionWorkerDependencies: %v", err)
|
||||
}
|
||||
if deps.guardrails == nil {
|
||||
t.Fatal("worker composition built no guardrail engine: TaskContext.Guardrails stays nil and no agent-stage decision is recorded")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTaskRecordsGuardrailDecision is the end-to-end half: a workflow run
|
||||
// whose TaskContext carries the engine records its agent stage's outcome. Before
|
||||
// the wiring the engine was never assigned, so the run recorded zero -- the
|
||||
// guardrail engine observed nothing for any task.
|
||||
func TestRunTaskRecordsGuardrailDecision(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
st := pgstore.Open(t)
|
||||
|
||||
task, err := st.EnqueueChatTask(ctx, "", "", "guardrail recording", "", "guardrail", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
if err := st.Transition(ctx, task.ID, task.Status, workflow.StatusRunning, "started"); err != nil {
|
||||
t.Fatalf("claim task: %v", err)
|
||||
}
|
||||
task.Status = workflow.StatusRunning
|
||||
task.WorkflowDefinitionYAML = guardrailRunYAML
|
||||
task.WorkflowDefinitionDigest = workflow.DigestDefinition(guardrailRunYAML)
|
||||
if err := st.Update(ctx, task); err != nil {
|
||||
t.Fatalf("seed task: %v", err)
|
||||
}
|
||||
|
||||
workDir := t.TempDir()
|
||||
// A threshold a single stage reaches turns the recorded outcome into an
|
||||
// actual guardrail decision, so the test proves the run consulted the
|
||||
// engine rather than merely holding a reference.
|
||||
engine := tools.NewGuardrailEngine(tools.ToolCallGuardrailConfig{
|
||||
NoProgressWarnAfter: 1,
|
||||
HardStopAfterWarnRepeat: 1,
|
||||
})
|
||||
request := taskrun.Request{
|
||||
Task: task,
|
||||
Repo: config.Repo{},
|
||||
Cfg: config.Config{Models: map[string]string{"builder": "p/m"}}.ForTask(),
|
||||
WorkflowDefinition: guardrailRunYAML,
|
||||
}
|
||||
dependencies := taskDependencies{
|
||||
store: st,
|
||||
trees: &fakeScratchTrees{dir: workDir},
|
||||
steps: testSteps(t),
|
||||
guardrails: engine,
|
||||
}
|
||||
fakeRunner := runnerFactory(func(map[string]agentexec.Provider, *slog.Logger) agentexec.Runner {
|
||||
return stubRunner{}
|
||||
})
|
||||
if _, err := runTask(ctx, request, dependencies, fakeRunner, workDir, slog.New(slog.DiscardHandler)); err != nil {
|
||||
t.Fatalf("runTask: %v", err)
|
||||
}
|
||||
|
||||
if got := engine.TotalCalls(); got != 1 {
|
||||
t.Fatalf("guardrail engine recorded %d tool outcomes, want 1 (the agent stage): TaskContext.Guardrails is not wired to the engine the run was given", got)
|
||||
}
|
||||
if !engine.HardStopped() {
|
||||
t.Fatal("guardrail engine issued no decision for the run's agent stage: TaskContext.Guardrails is not wired to the engine the run was given")
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/installtype"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
"github.com/samcharles93/archie-core/internal/worktree"
|
||||
)
|
||||
|
||||
@@ -186,6 +187,11 @@ type taskDependencies struct {
|
||||
// executing half of the contract whose validating half is the control
|
||||
// plane's (archie-core-fwmp).
|
||||
steps *workflow.Manager
|
||||
// guardrails is the guardrail engine agent stages record tool outcomes
|
||||
// against. It is built by this process's composition root (the workflow
|
||||
// runs in archie-agent, so the daemon's engine cannot reach it) and set on
|
||||
// every TaskContext; nil means guardrails are disabled.
|
||||
guardrails *tools.GuardrailEngine
|
||||
}
|
||||
|
||||
type runnerFactory func(map[string]agentexec.Provider, *slog.Logger) agentexec.Runner
|
||||
@@ -312,16 +318,17 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen
|
||||
}
|
||||
|
||||
tc := &workflow.TaskContext{
|
||||
Task: req.Task,
|
||||
Repo: req.Repo,
|
||||
Cfg: req.Cfg.ToConfig(),
|
||||
Forge: dependencies.forge,
|
||||
Store: dependencies.store,
|
||||
Calls: dependencies.calls,
|
||||
Trees: trees,
|
||||
Agent: agent,
|
||||
Bus: bus,
|
||||
Log: log,
|
||||
Task: req.Task,
|
||||
Repo: req.Repo,
|
||||
Cfg: req.Cfg.ToConfig(),
|
||||
Forge: dependencies.forge,
|
||||
Store: dependencies.store,
|
||||
Calls: dependencies.calls,
|
||||
Trees: trees,
|
||||
Agent: agent,
|
||||
Bus: bus,
|
||||
Log: log,
|
||||
Guardrails: dependencies.guardrails,
|
||||
}
|
||||
// pr-review reads an external pull request's data from workDir, which the
|
||||
// daemon already populated before this task started (daemon's
|
||||
|
||||
@@ -696,7 +696,7 @@ func TestExecuteTaskRequestUsesInfrastructureRPCDependencies(t *testing.T) {
|
||||
Cfg: config.Config{}.ForTask(),
|
||||
WorktreeGrant: grant,
|
||||
}
|
||||
response, err := executeTaskRequest(ctx, request, transport, hostDir, slog.New(slog.DiscardHandler), testSteps(t))
|
||||
response, err := executeTaskRequest(ctx, request, transport, hostDir, slog.New(slog.DiscardHandler), testSteps(t), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -814,7 +814,7 @@ func TestExecuteTaskRequestForwardsWorkflowEventsOverNATS(t *testing.T) {
|
||||
Cfg: config.Config{}.ForTask(),
|
||||
WorktreeGrant: grant,
|
||||
}
|
||||
response, err := executeTaskRequest(ctx, request, transport, hostDir, slog.New(slog.DiscardHandler), testSteps(t))
|
||||
response, err := executeTaskRequest(ctx, request, transport, hostDir, slog.New(slog.DiscardHandler), testSteps(t), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps"
|
||||
"github.com/samcharles93/archie-core/internal/storage"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
)
|
||||
|
||||
const rpcTimeout = 60 * time.Second
|
||||
@@ -64,6 +65,11 @@ type workerDependencies struct {
|
||||
// definitions against, registered by the composition root before the first
|
||||
// task is served (infrastructure/workflowsteps is the provider set).
|
||||
steps *workflow.Manager
|
||||
// guardrails is the guardrail engine every served task records agent-stage
|
||||
// outcomes against, built once at this process's composition root.
|
||||
// archie-agent is the process that runs the stages, so the engine that
|
||||
// observes them lives here; a live engine cannot cross the task handoff.
|
||||
guardrails *tools.GuardrailEngine
|
||||
}
|
||||
|
||||
// productionWorkerDependencies builds the worker's process dependencies. The
|
||||
@@ -87,7 +93,8 @@ func productionWorkerDependencies() (workerDependencies, error) {
|
||||
<-ctx.Done()
|
||||
log.Info("archie-agent shutting down")
|
||||
},
|
||||
steps: steps,
|
||||
steps: steps,
|
||||
guardrails: tools.NewGuardrailEngine(tools.DefaultGuardrailConfig()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -134,7 +141,7 @@ func run(ctx context.Context, settings Settings, log *slog.Logger, dependencies
|
||||
log.Info("system log publisher attached", "task", taskID)
|
||||
|
||||
subscription, err := transport.SubscribeTasks(ctx, taskID, func(ctx context.Context, request taskrun.Request) (*taskrun.Response, error) {
|
||||
return executeTaskRequest(ctx, request, transport, workDir, log, dependencies.steps)
|
||||
return executeTaskRequest(ctx, request, transport, workDir, log, dependencies.steps, dependencies.guardrails)
|
||||
}, log)
|
||||
if err != nil {
|
||||
log.Error("taskrun subscribe failed", "err", err)
|
||||
@@ -159,15 +166,16 @@ type taskServiceTransport interface {
|
||||
EventPublisher() agentexec.EventPublisher
|
||||
}
|
||||
|
||||
func executeTaskRequest(ctx context.Context, request taskrun.Request, transport taskServiceTransport, workDir string, log *slog.Logger, steps *workflow.Manager) (*taskrun.Response, error) {
|
||||
func executeTaskRequest(ctx context.Context, request taskrun.Request, transport taskServiceTransport, workDir string, log *slog.Logger, steps *workflow.Manager, guardrails *tools.GuardrailEngine) (*taskrun.Response, error) {
|
||||
log.Info("running task", "task", request.Task.ID, "repo", request.Repo.FullName(), "issue", request.Task.IssueNumber)
|
||||
dependencies := taskDependencies{
|
||||
forge: transport.Forger(request.Task.Identity, rpcTimeout),
|
||||
store: transport.Store(rpcTimeout),
|
||||
calls: transport.Calls(rpcTimeout),
|
||||
trees: transport.Trees(request.Task.Identity, request.WorktreeGrant, rpcTimeout),
|
||||
events: transport.EventPublisher(),
|
||||
steps: steps,
|
||||
forge: transport.Forger(request.Task.Identity, rpcTimeout),
|
||||
store: transport.Store(rpcTimeout),
|
||||
calls: transport.Calls(rpcTimeout),
|
||||
trees: transport.Trees(request.Task.Identity, request.WorktreeGrant, rpcTimeout),
|
||||
events: transport.EventPublisher(),
|
||||
steps: steps,
|
||||
guardrails: guardrails,
|
||||
}
|
||||
response, err := runTask(ctx, request, dependencies, newTaskRunner, workDir, log)
|
||||
if err != nil {
|
||||
|
||||
@@ -287,7 +287,6 @@ type boot struct {
|
||||
memEngines *domainmemory.Registry
|
||||
curatorRegistry *curator.Registry
|
||||
curatorRuntime *curator.Runtime
|
||||
guardrails *tools.GuardrailEngine
|
||||
providerRegistry *toolprovider.Registry
|
||||
// mcpMu guards mcpApplied, which records the MCP servers the provider
|
||||
// registry was last built from, keyed by configured name. A live
|
||||
@@ -1400,18 +1399,6 @@ func (b *boot) setupCurators(ctx context.Context) {
|
||||
b.addCleanup(shutdownCuratorRegistry(reg, log))
|
||||
}
|
||||
|
||||
func (b *boot) setupGuardrails() {
|
||||
log := b.log
|
||||
gc := tools.DefaultGuardrailConfig()
|
||||
b.guardrails = tools.NewGuardrailEngine(gc)
|
||||
log.Info(
|
||||
"guardrail engine enabled",
|
||||
"exact_failure_warn", gc.ExactFailureWarnAfter,
|
||||
"same_tool_failure_warn", gc.SameToolFailureWarnAfter,
|
||||
"no_progress_warn", gc.NoProgressWarnAfter,
|
||||
)
|
||||
}
|
||||
|
||||
// registerTools registers the tool providers with a lifecycle: workspace
|
||||
// file/shell tools and optional MCP servers. Memory tools
|
||||
// (memory_create/update/delete/list) are not registered here -- they are
|
||||
@@ -1610,7 +1597,6 @@ func (b *boot) buildDaemon() {
|
||||
StateStoreGrants: b.stateStoreGrants,
|
||||
ContainerPool: b.containerPool,
|
||||
KitLauncher: b.kitLauncher,
|
||||
Guardrails: b.guardrails,
|
||||
ToolRegistry: b.toolReg,
|
||||
Identities: b.identityRunners,
|
||||
RootIdentityID: identity.StableID(configuredIdentityNames(b.cfg)[0]),
|
||||
|
||||
@@ -425,7 +425,6 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
return 1
|
||||
}
|
||||
|
||||
b.setupGuardrails()
|
||||
if err := b.registerTools(ctx); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -225,12 +225,6 @@ type Daemon struct {
|
||||
IdentityRepository identity.Repository
|
||||
RootIdentityID identity.IdentityID
|
||||
|
||||
// Guardrails is the tool-call guardrail engine, wired by the composition
|
||||
// root. When non-nil, tool successes and failures are recorded and
|
||||
// warnings/hard-stops are issued per the configured thresholds. Nil
|
||||
// means guardrails are disabled (backward compatible).
|
||||
Guardrails *tools.GuardrailEngine
|
||||
|
||||
// KindWorkflows and LabelWorkflows are the resolved kind/label ->
|
||||
// workflow-name routing bindings loaded at startup (WorkflowRoutingFile,
|
||||
// WorkflowLabelsFile, PlaybookDirs). They travel in taskrun.Request so
|
||||
|
||||
@@ -190,7 +190,9 @@ type TaskContext struct {
|
||||
|
||||
// Guardrails is the per-task guardrail engine reference. When non-nil,
|
||||
// agent stages record tool successes and failures for guardrail
|
||||
// enforcement. Wired by the composition root from the daemon.
|
||||
// enforcement. Wired by the composition root of the process that runs the
|
||||
// workflow (archie-agent), because the engine has to live where the agent
|
||||
// stages execute.
|
||||
Guardrails *tools.GuardrailEngine
|
||||
|
||||
// RunUsage accumulates every agent run's token breakdown (prompt,
|
||||
|
||||
Reference in new issue
Block a user