Files
archie/internal/agentexec/capture_mcp_test.go
T
sam 37d913c5fd feat(agentexec): serve capture tools to harnesses over MCP
archie-agent mcp is a stdio MCP server, on the official go-sdk, serving a
stage's capture tools and appending accepted calls to a captures file.
The harness runner writes the tool spec and a standard mcpServers config,
registers it through HarnessSpec.MCPConfig, and reads the captures back
into the stage result. The captures file is writable by the harness user,
so the worker validates every record again with the same acceptCapture
the built-in loop now shares, and forged, undeclared or over-cap records
are dropped. A stage with capture tools on a harness without MCP is
refused before it runs.
2026-09-26 03:28:58 +10:00

151 lines
4.8 KiB
Go

package agentexec
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
var findingTool = CaptureTool{
Name: "record_finding",
Description: "Record one finding.",
Parameters: json.RawMessage(`{"type":"object","properties":{"title":{"type":"string"},"severity":{"type":"string"}},"required":["title"]}`),
RequiredFields: []string{"title"},
MaxCalls: 2,
}
type lockedBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (b *lockedBuffer) Write(p []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.Write(p)
}
func (b *lockedBuffer) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.String()
}
func connectCaptureServer(t *testing.T, sink *lockedBuffer) *mcp.ClientSession {
t.Helper()
serverT, clientT := mcp.NewInMemoryTransports()
go func() { _ = ServeCaptureMCP(t.Context(), []CaptureTool{findingTool}, sink, serverT) }()
session, err := mcp.NewClient(&mcp.Implementation{Name: "test", Version: "0"}, nil).Connect(t.Context(), clientT, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = session.Close() })
return session
}
func callText(t *testing.T, s *mcp.ClientSession, args any) string {
t.Helper()
res, err := s.CallTool(t.Context(), &mcp.CallToolParams{Name: "record_finding", Arguments: args})
if err != nil {
t.Fatal(err)
}
var out strings.Builder
for _, c := range res.Content {
if text, ok := c.(*mcp.TextContent); ok {
out.WriteString(text.Text)
}
}
return out.String()
}
func TestCaptureMCPServesTheStageCaptureTools(t *testing.T) {
sink := &lockedBuffer{}
s := connectCaptureServer(t, sink)
tools, err := s.ListTools(t.Context(), nil)
if err != nil {
t.Fatal(err)
}
if len(tools.Tools) != 1 || tools.Tools[0].Name != "record_finding" || tools.Tools[0].Description != "Record one finding." {
t.Fatalf("tools %+v, want the stage's one capture tool", tools.Tools)
}
if got := callText(t, s, map[string]any{"title": "nil deref", "severity": "important"}); !strings.Contains(got, "recorded") {
t.Fatalf("valid call replied %q", got)
}
if got := callText(t, s, map[string]any{"severity": "nitpick"}); !strings.Contains(got, "rejected") {
t.Fatalf("call missing a required field replied %q", got)
}
if got := callText(t, s, map[string]any{"title": "second"}); !strings.Contains(got, "recorded") {
t.Fatalf("second valid call replied %q", got)
}
if got := callText(t, s, map[string]any{"title": "third"}); !strings.Contains(got, "maximum") {
t.Fatalf("call past MaxCalls replied %q", got)
}
lines := strings.Split(strings.TrimSpace(sink.String()), "\n")
if len(lines) != 2 {
t.Fatalf("sink has %d records, want only the 2 accepted calls:\n%s", len(lines), sink.String())
}
}
func TestReadCapturesRevalidatesEverything(t *testing.T) {
// The captures file is writable by the harness user, so anything in it
// may be forged. Only records that pass the same validation survive.
path := filepath.Join(t.TempDir(), "captures.jsonl")
content := strings.Join([]string{
`{"tool":"record_finding","args":{"title":"real"}}`,
`{"tool":"record_finding","args":{"severity":"no title"}}`,
`{"tool":"grant_admin","args":{"title":"x"}}`,
`not json at all`,
`{"tool":"record_finding","args":{"title":"second"}}`,
`{"tool":"record_finding","args":{"title":"over the cap"}}`,
}, "\n")
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
captures, err := readCaptures(path, []CaptureTool{findingTool})
if err != nil {
t.Fatal(err)
}
var titles []string
for _, raw := range captures["record_finding"] {
var v struct{ Title string }
_ = json.Unmarshal(raw, &v)
titles = append(titles, v.Title)
}
if !slices.Equal(titles, []string{"real", "second"}) || len(captures) != 1 {
t.Fatalf("captures %v (titles %v), want only the two valid record_finding calls", captures, titles)
}
}
func TestHarnessCollectsCapturesThroughTheMCPServer(t *testing.T) {
f := newFixture(t, "capture")
f.req.CaptureTools = []CaptureTool{findingTool}
f.req.Harness.MCPConfig = []string{"--mcp-config", "{{.MCPConfig}}"}
res, _ := f.run(t, t.Context())
if res.Status != StatusPassed {
t.Fatalf("status %q (%s)", res.Status, res.Detail)
}
if got := len(res.Captures["record_finding"]); got != 1 {
t.Fatalf("captures %v, want the one valid record the harness made", res.Captures)
}
inv := f.invocations(t)
if len(inv) != 1 || !strings.Contains(inv[0], "--mcp-config ") {
t.Fatalf("invocations %q, want the MCP config registered", inv)
}
}
func TestHarnessRefusesCaptureToolsWithoutMCP(t *testing.T) {
f := newFixture(t, "edit")
f.req.CaptureTools = []CaptureTool{findingTool}
if _, err := NewHarnessRunner(nil).Run(t.Context(), f.workspace, f.req, nil); err == nil {
t.Fatal("a stage with capture tools ran on a harness with no MCP registration; its results would be lost")
}
}