Files
archie/internal/agentexec/loop_runner_test.go
T
sam c431834e2b feat(eda): workflow inputs, optional repository and agent profiles
Workflow definitions declare inputs, repository (none, optional or
required; default required) and profile. A binding assigns each input
from a mapped parameter or a constant and may take its repository from a
parameter, which must name a configured repo; both are checked when the
binding is saved and again at dispatch. Inputs travel as structured task
data (tasks.inputs, the task brief, and a JSON block in the agent's
mission), never as body text.

A task with no repository runs in a per-task scratch workspace with no
clone and no publication grant. The agent.run step runs one mission
without a repository and finishes the task in the new completed status.

Agent profiles live under [containers.profiles.<name>]: an image and an
allowlist over the tools archie adds (MCP, scripts, plugins).
[containers].image is the default profile; an unconfigured profile parks
the task. The workflow is now pinned before the container starts so the
profile can choose its image.

Also stop two archied tests writing embedded NATS state into the package
directory, refuse an empty state_dir for embedded NATS, and drop the NATS
store files an earlier lane commit tracked.
2026-09-24 20:29:33 +10:00

393 lines
13 KiB
Go

package agentexec
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"github.com/samcharles93/ai-sdk/agentloop"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/ai-sdk/runtime"
"github.com/samcharles93/archie-core/internal/tools"
)
func TestPluginToolSetExecutesBundledPlugin(t *testing.T) {
set, err := pluginToolSet(Request{Plugins: []PluginSpec{{Name: "echo", Src: "package main\nfunc Run(input string) string { return input + \"!\" }"}}}, t.TempDir())
if err != nil {
t.Fatal(err)
}
tool := set["echo"]
got, err := tool.Execute(context.Background(), `{"input":"hello"}`)
if err != nil {
t.Fatal(err)
}
if got != "hello!" {
t.Errorf("plugin output = %q, want hello!", got)
}
}
func TestPluginToolSetExcludedFromConstrainedStages(t *testing.T) {
requests := []Request{
{ReadOnly: true, Plugins: []PluginSpec{{Name: "unsafe"}}},
{Protection: Protection{Suffixes: []string{"_test.go"}}, Plugins: []PluginSpec{{Name: "unsafe"}}},
{Gate: Gate{Commands: []Command{{Name: "test"}}}, Plugins: []PluginSpec{{Name: "unsafe"}}},
}
for _, request := range requests {
set, err := pluginToolSet(request, t.TempDir())
if err != nil {
t.Fatal(err)
}
if len(set) != 0 {
t.Error("plugin tool exposed in constrained stage")
}
}
}
func TestPluginToolSetRejectsAgentLoopCollisions(t *testing.T) {
for _, name := range []string{"write", "finish"} {
_, err := pluginToolSet(Request{Plugins: []PluginSpec{{Name: name}}}, t.TempDir())
if err == nil {
t.Errorf("plugin %q did not conflict with agent-loop tool", name)
}
}
}
func TestLoopRunnerMapsRequestAndCapturesOutput(t *testing.T) {
runner := &LoopRunner{
runtime: runtime.NewRuntime(runtime.Config{}),
run: func(ctx context.Context, cfg agentloop.Config) (agentloop.Result, error) {
if cfg.WorkDir != "/workspace" || cfg.ModelRef != "provider/model" || cfg.Mission != "mission" {
t.Fatalf("unexpected config: workdir=%q model=%q mission=%q", cfg.WorkDir, cfg.ModelRef, cfg.Mission)
}
if cfg.ProtectPaths == nil || !cfg.ProtectPaths("nested/file_test.go") || !cfg.ProtectPaths("view_templ.go") {
t.Fatal("declarative path protection was not applied")
}
if cfg.Completion != agentloop.CompletionForceFinish {
t.Fatalf("Completion = %q, want forced finish recovery", cfg.Completion)
}
if !cfg.Compact.Enabled || cfg.Compact.ContextWindow <= 0 {
t.Fatalf("Compact = %#v, want adaptive non-blocking history compaction", cfg.Compact)
}
if !strings.Contains(cfg.ExtraRules, "Confine discovery to /workspace") ||
!strings.Contains(cfg.ExtraRules, "unless the mission explicitly requests") {
t.Fatalf("project discovery rule missing from ExtraRules: %q", cfg.ExtraRules)
}
if err := cfg.Notes.Append(ctx, "verified note"); err != nil {
t.Fatal(err)
}
if _, err := cfg.Extra["decide"].Execute(ctx, `{"fit":true}`); err != nil {
t.Fatal(err)
}
return agentloop.Result{
Status: agentloop.StatusPassed, Summary: "done", TokensUsed: 10,
Usage: chat.Usage{PromptTokens: 8, CompletionTokens: 2, TotalTokens: 10, CachedTokens: 6},
}, nil
},
}
req := Request{
Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "assess", Model: "provider/model", ContextWindow: 120_000, Mission: "mission",
Protection: Protection{Suffixes: []string{"_templ.go"}, Globs: []string{"*_test.go"}},
CaptureTools: []CaptureTool{{Name: "decide", Parameters: json.RawMessage(`{"type":"object"}`), MaxCalls: 1}},
}
got, err := runner.Run(context.Background(), "/workspace", req, nil)
if err != nil {
t.Fatal(err)
}
if got.Status != StatusPassed || got.Summary != "done" || got.TokensUsed != 10 {
t.Fatalf("unexpected result: %#v", got)
}
if got.Usage.PromptTokens != 8 || got.Usage.CompletionTokens != 2 || got.Usage.CachedTokens != 6 {
t.Fatalf("usage = %#v, want provider breakdown", got.Usage)
}
if len(got.AppendedNotes) != 1 || got.AppendedNotes[0] != "verified note" {
t.Fatalf("appended notes = %v", got.AppendedNotes)
}
if len(got.Captures["decide"]) != 1 || string(got.Captures["decide"][0]) != `{"fit":true}` {
t.Fatalf("captures = %v", got.Captures)
}
}
func TestLoopRunnerIncludesAvailableCentralTools(t *testing.T) {
registry := tools.NewRegistry()
if err := registry.Register(tools.ToolEntry{
Name: "memory_search",
Handler: func(context.Context, map[string]any) (any, error) {
return "found", nil
},
}); err != nil {
t.Fatal(err)
}
runner := &LoopRunner{
runtime: runtime.NewRuntime(runtime.Config{}),
tools: registry,
run: func(ctx context.Context, cfg agentloop.Config) (agentloop.Result, error) {
tool, ok := cfg.Extra["memory_search"]
if !ok {
t.Fatal("central tool is not present in agentloop Config.Extra")
}
got, err := tool.Execute(ctx, `{}`)
if err != nil {
t.Fatal(err)
}
if got != `"found"` {
t.Fatalf("tool output = %q, want JSON string", got)
}
return agentloop.Result{Status: agentloop.StatusPassed}, nil
},
}
_, err := runner.Run(context.Background(), "/workspace", Request{
Version: ProtocolVersion,
TaskID: 1,
Attempt: 1,
Stage: "build",
Model: "provider/model",
Mission: "mission",
}, nil)
if err != nil {
t.Fatal(err)
}
}
func TestLoopRunnerRejectsInvalidCentralToolSchemas(t *testing.T) {
registry := tools.NewRegistry()
if err := registry.Register(tools.ToolEntry{
Name: "invalid",
Handler: func(context.Context, map[string]any) (any, error) { return nil, nil },
DynamicSchemaOverrides: func(tools.JSONSchema) tools.JSONSchema {
panic("schema panic")
},
}); err != nil {
t.Fatal(err)
}
runner := &LoopRunner{
runtime: runtime.NewRuntime(runtime.Config{}),
tools: registry,
run: func(context.Context, agentloop.Config) (agentloop.Result, error) {
t.Fatal("agent loop ran with an invalid central tool schema")
return agentloop.Result{}, nil
},
}
_, err := runner.Run(context.Background(), "/workspace", Request{
Version: ProtocolVersion,
TaskID: 1,
Attempt: 1,
Stage: "build",
Model: "provider/model",
Mission: "mission",
}, nil)
if err == nil {
t.Fatal("Run() error = nil, want invalid central tool schema error")
}
}
func TestProtectionMatcherAppliesBasenameAndPathGlobs(t *testing.T) {
match := protectionMatcher(Protection{Globs: []string{"*_test.go", "tests/*.rs"}}, false)
tests := []struct {
path string
want bool
}{
{path: "main_test.go", want: true},
{path: "internal/foo/bar_test.go", want: true},
{path: "tests/widget.rs", want: true},
{path: "src/widget.rs", want: false},
}
for _, tt := range tests {
if got := match(tt.path); got != tt.want {
t.Errorf("match(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestCaptureToolRejectsInvalidDataBeforeRecording(t *testing.T) {
captures := make(map[string][]json.RawMessage)
tool := captureToolSet([]CaptureTool{{
Name: "decide", RequiredFields: []string{"fit", "reasons"}, NonEmptyStrings: []string{"reasons"},
BooleanFields: []string{"fit"},
}}, captures)["decide"]
output, err := tool.Execute(context.Background(), `{"fit":true,"reasons":""}`)
if err != nil {
t.Fatal(err)
}
if len(captures["decide"]) != 0 || output != "decide rejected: reasons must be a non-empty string" {
t.Fatalf("invalid capture output=%q captures=%v", output, captures)
}
}
func TestCaptureToolRejectsWrongFieldTypeBeforeRecording(t *testing.T) {
captures := make(map[string][]json.RawMessage)
tool := captureToolSet([]CaptureTool{{Name: "decide", BooleanFields: []string{"fit"}}}, captures)["decide"]
output, err := tool.Execute(context.Background(), `{"fit":"true"}`)
if err != nil {
t.Fatal(err)
}
if len(captures["decide"]) != 0 || output != "decide rejected: fit must be a boolean" {
t.Fatalf("invalid capture output=%q captures=%v", output, captures)
}
}
func TestScriptToolRunsAGoScriptAndReturnsItsOutput(t *testing.T) {
workspace := t.TempDir()
if err := os.MkdirAll(filepath.Join(workspace, "scripts"), 0o755); err != nil {
t.Fatal(err)
}
src := "package main\n\nimport \"fmt\"\n\nfunc main() { fmt.Println(\"hi from a script\") }\n"
if err := os.WriteFile(filepath.Join(workspace, "scripts", "hello.go"), []byte(src), 0o644); err != nil {
t.Fatal(err)
}
tool := scriptToolSet(workspace)["run_go_script"]
out, err := tool.Execute(context.Background(), `{"path":"scripts/hello.go"}`)
if err != nil {
t.Fatal(err)
}
if out != "hi from a script\n" {
t.Fatalf("run_go_script output = %q", out)
}
}
func TestScriptToolRejectsPathsEscapingTheWorkspace(t *testing.T) {
workspace := t.TempDir()
tool := scriptToolSet(workspace)["run_go_script"]
out, err := tool.Execute(context.Background(), `{"path":"../outside.go"}`)
if err != nil {
t.Fatal(err)
}
if out != "run_go_script rejected: path escapes the workspace" {
t.Fatalf("run_go_script output = %q, want a workspace-escape rejection", out)
}
}
func TestScriptToolRejectsMissingPath(t *testing.T) {
tool := scriptToolSet(t.TempDir())["run_go_script"]
out, err := tool.Execute(context.Background(), `{}`)
if err != nil {
t.Fatal(err)
}
if out != "run_go_script rejected: arguments must be a JSON object with a non-empty path field" {
t.Fatalf("run_go_script output = %q", out)
}
}
func TestMergeToolSetsLaterWinsOnCollision(t *testing.T) {
first := scriptToolSet(t.TempDir())
captures := make(map[string][]json.RawMessage)
second := captureToolSet([]CaptureTool{{Name: "run_go_script"}}, captures)
merged := mergeToolSets(first, second)
if len(merged) != 1 {
t.Fatalf("merged = %v, want exactly one tool", merged)
}
if _, err := merged["run_go_script"].Execute(context.Background(), `{}`); err != nil {
t.Fatal(err)
}
if len(captures["run_go_script"]) != 1 {
t.Fatalf("expected the capture tool (second set) to win, captures = %v", captures)
}
}
func TestLoopRunnerPreservesCallerCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
runner := &LoopRunner{
runtime: runtime.NewRuntime(runtime.Config{}),
run: func(context.Context, agentloop.Config) (agentloop.Result, error) {
return agentloop.Result{Status: agentloop.StatusParked, StopReason: agentloop.StopTimedOut}, nil
},
}
_, err := runner.Run(ctx, "/workspace", Request{
Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "build", Model: "provider/model", Mission: "mission",
}, nil)
if !errors.Is(err, context.Canceled) {
t.Fatalf("Run error = %v, want context.Canceled", err)
}
}
// TestScriptToolHonorsContextCancellation proves that the run_go_script
// tool handler discards the incoming context (the handler is declared as
// func(_ context.Context, input string) and calls skillscript.Run without
// forwarding ctx). Every other execution path in archie-core respects
// context cancellation (LoopRunner checks context.Cause), but a
// cancelled/timed-out context
// has zero effect on run_go_script — a blocking script wedges the
// goroutine permanently.
func TestScriptToolHonorsContextCancellation(t *testing.T) {
workspace := t.TempDir()
src := `package main
func main() {
select {}
}
`
if err := os.WriteFile(filepath.Join(workspace, "blocker.go"), []byte(src), 0o644); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel() // immediately cancelled — the tool should return promptly
tool := scriptToolSet(workspace)["run_go_script"]
done := make(chan struct{})
go func() {
// The result is irrelevant here -- this goroutine only exists to
// prove Execute returns promptly once ctx is cancelled.
_, _ = tool.Execute(ctx, `{"path":"blocker.go"}`)
close(done)
}()
select {
case <-done:
// After the fix, the tool returns quickly because the cancelled
// context propagates through to skillscript.Run, which selects
// on ctx.Done() and abandons the blocking EvalPath goroutine.
t.Log("run_go_script returned promptly (context was honored)")
case <-time.After(3 * time.Second):
t.Error("run_go_script ignores context cancellation: " +
"the handler discards its context parameter (declared as _) " +
"and calls skillscript.Run without forwarding it (issue #45)")
}
}
// A profile's allowlist removes the tools archie adds that it does not name,
// but never a stage's capture tools, which carry its structured result.
func TestLoopRunnerAppliesTheProfileAllowlist(t *testing.T) {
registry := tools.NewRegistry()
for _, name := range []string{"whois", "memory_search"} {
if err := registry.Register(tools.ToolEntry{
Name: name,
Handler: func(context.Context, map[string]any) (any, error) { return "", nil },
}); err != nil {
t.Fatal(err)
}
}
var extra []string
runner := &LoopRunner{
runtime: runtime.NewRuntime(runtime.Config{}),
tools: registry,
AllowTools: []string{"whois"},
run: func(_ context.Context, cfg agentloop.Config) (agentloop.Result, error) {
for name := range cfg.Extra {
extra = append(extra, name)
}
return agentloop.Result{Status: agentloop.StatusPassed}, nil
},
}
_, err := runner.Run(context.Background(), t.TempDir(), Request{
Version: ProtocolVersion, TaskID: 1, Attempt: 1, Stage: "build", Model: "provider/model", Mission: "mission",
CaptureTools: []CaptureTool{{Name: "report", Description: "d", Parameters: json.RawMessage(`{"type":"object"}`)}},
}, nil)
if err != nil {
t.Fatal(err)
}
slices.Sort(extra)
if !slices.Equal(extra, []string{"report", "whois"}) {
t.Fatalf("agent tools = %v, want the allowed whois and the capture tool", extra)
}
}