Files
archie/internal/app/agentworker/review_test.go
T
sam bdf73ed371 feat(workflow): review calibration -- checked properties, style nits out of contract
Closes archie-core-h019.5. The shipped contract already made only a confirmed
error-level finding block, and kept zero-findings distinct from not-run; what
was missing was the other half of calibration, and that is what a reviewer that
cries wolf needs.

- ReviewReport gains Checked []ReviewCheck {Property, Evidence}: the properties
  the reviewer verified clean. Evidence is required, for the same reason a
  finding requires a failure scenario -- a check without evidence is an
  assertion, not a verification.
- The reviewer records checks through a typed record_checked tool, mirroring
  record_finding, so it cannot end its turn having silently skipped the
  checklist.
- Style/formatting/naming nits are stated out of contract, in the ReviewFinding
  type doc and the reviewer prompt: they are the padding that trains an
  operator to skim.
- The PR-body findings section renders the cleared properties, so a
  zero-finding review arrives backed by what was checked rather than reading
  the same as a review that never looked.

Design: docs/prds/review-calibration.md.
2026-09-11 20:18:06 +10:00

253 lines
9.1 KiB
Go

package agentworker
import (
"context"
"errors"
"os"
"path/filepath"
"slices"
"testing"
"github.com/samcharles93/ai-sdk/agent"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/archie-core/internal/domain/workflow"
)
// TestReviewerToolSetIsReadOnlyAndHasNoMutationPath asserts, as a
// construction fact rather than a prompt claim, that the reviewer's
// toolset carries no tool capable of writing, editing, or running a
// shell command -- the class of tool that could reach the worktreerpc
// publication grant or otherwise mutate anything. Read/grep/find plus the
// findings-capture tool are the entire surface.
func TestReviewerToolSetIsReadOnlyAndHasNoMutationPath(t *testing.T) {
var findings []workflow.ReviewFinding
var checks []workflow.ReviewCheck
set := reviewerToolSet(t.TempDir(), &findings, &checks)
names := make([]string, 0, len(set))
for name := range set {
names = append(names, name)
}
slices.Sort(names)
want := []string{"find", "grep", "read", "record_checked", "record_finding"}
slices.Sort(want)
if !slices.Equal(names, want) {
t.Fatalf("reviewer toolset = %v, want exactly %v (no write/edit/shell)", names, want)
}
}
// TestReviewerToolSetIsRootedAtTheGivenSnapshot asserts the read tool
// cannot read outside the snapshot directory it was constructed with --
// it has no path back to the implementer's actual worktree.
func TestReviewerToolSetIsRootedAtTheGivenSnapshot(t *testing.T) {
snapshot := t.TempDir()
if err := os.WriteFile(filepath.Join(snapshot, "in-snapshot.txt"), []byte("ok\n"), 0o644); err != nil {
t.Fatal(err)
}
outside := t.TempDir()
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("nope\n"), 0o644); err != nil {
t.Fatal(err)
}
var findings []workflow.ReviewFinding
var checks []workflow.ReviewCheck
set := reviewerToolSet(snapshot, &findings, &checks)
readTool := set["read"]
if readTool == nil {
t.Fatal("reviewer toolset has no read tool")
}
out, err := readTool.Execute(t.Context(), `{"path":"in-snapshot.txt"}`)
if err != nil {
t.Fatalf("read in-snapshot file: %v", err)
}
if !containsSubstr(out, "ok") {
t.Errorf("read in-snapshot file = %q, want it to contain the file's content", out)
}
// Tool failures are encoded in-band in the returned content, not as a
// Go error (toolkit.Registry.CoreToolSet's own contract) -- an escape
// attempt must reject there, not succeed.
escapeOut, err := readTool.Execute(t.Context(), `{"path":"`+filepath.Join(outside, "secret.txt")+`"}`)
if err != nil {
t.Fatalf("read outside-snapshot path: %v", err)
}
if containsSubstr(escapeOut, "nope") {
t.Errorf("read tool returned content from outside the snapshot: %q", escapeOut)
}
if !containsSubstr(escapeOut, "escapes") {
t.Errorf("read tool output = %q, want a path-confinement rejection", escapeOut)
}
}
// TestRecordFindingToolAppendsToCapturedSlice asserts the capture-tool
// contract: findings are recorded via the closure over the findings
// slice, not returned from Subagent.Run's text.
func TestRecordFindingToolAppendsToCapturedSlice(t *testing.T) {
var findings []workflow.ReviewFinding
tool := recordFindingTool(&findings)
input := `{"file":"main.go","line":10,"defect":"nil deref","failure_scenario":"calling Foo(nil) panics","verdict":"confirmed","level":"error","category":"nil-risk"}`
if _, err := tool.Execute(t.Context(), input); err != nil {
t.Fatalf("record_finding: %v", err)
}
if len(findings) != 1 {
t.Fatalf("captured %d findings, want 1", len(findings))
}
if !findings[0].Blocking() {
t.Error("captured finding is not blocking, want confirmed+error to block")
}
}
// TestRecordFindingToolRejectsInvalidVerdictWithoutError asserts an
// invalid finding is rejected in-band (so the model can retry) rather
// than surfacing as a hard tool error, and is not captured.
func TestRecordFindingToolRejectsInvalidVerdictWithoutError(t *testing.T) {
var findings []workflow.ReviewFinding
tool := recordFindingTool(&findings)
input := `{"file":"main.go","line":1,"defect":"x","failure_scenario":"y","verdict":"maybe","level":"error","category":"other"}`
out, err := tool.Execute(t.Context(), input)
if err != nil {
t.Fatalf("record_finding returned a Go error for invalid input: %v", err)
}
if !containsSubstr(out, "rejected") {
t.Errorf("record_finding output = %q, want a rejection message", out)
}
if len(findings) != 0 {
t.Errorf("captured %d findings, want 0 for an invalid finding", len(findings))
}
}
// TestRecordCheckedToolAppendsToCapturedSlice asserts the calibration capture
// contract: a cleared property is recorded through the closure, so a
// zero-finding review can report what it actually checked.
func TestRecordCheckedToolAppendsToCapturedSlice(t *testing.T) {
var checks []workflow.ReviewCheck
tool := recordCheckedTool(&checks)
input := `{"property":"nil-safety of Foo callers","evidence":"read all 4 callers; each nil-checks"}`
if _, err := tool.Execute(t.Context(), input); err != nil {
t.Fatalf("record_checked: %v", err)
}
if len(checks) != 1 || checks[0].Property != "nil-safety of Foo callers" {
t.Fatalf("captured checks = %+v, want one check for the recorded property", checks)
}
}
// TestRecordCheckedToolRejectsMissingEvidence asserts a check without evidence
// is rejected in-band and not captured, so "verified clean" cannot be recorded
// as a bare assertion.
func TestRecordCheckedToolRejectsMissingEvidence(t *testing.T) {
var checks []workflow.ReviewCheck
tool := recordCheckedTool(&checks)
out, err := tool.Execute(t.Context(), `{"property":"looks fine","evidence":""}`)
if err != nil {
t.Fatalf("record_checked returned a Go error for invalid input: %v", err)
}
if !containsSubstr(out, "rejected") {
t.Errorf("record_checked output = %q, want a rejection message", out)
}
if len(checks) != 0 {
t.Errorf("captured %d checks, want 0 for a check with no evidence", len(checks))
}
}
// TestCompletedReviewCarriesChecks asserts the report the reviewer returns
// carries the captured checks, not just the findings.
func TestCompletedReviewCarriesChecks(t *testing.T) {
checks := []workflow.ReviewCheck{{Property: "p", Evidence: "e"}}
report := completedReview(nil, checks, "summary")
if !report.Ran() {
t.Fatal("completedReview report did not run")
}
if len(report.Checked) != 1 || report.Checked[0].Property != "p" {
t.Fatalf("report.Checked = %+v, want the captured check", report.Checked)
}
}
// TestReviewSubagentConstructionNeverSendsConversationHistory asserts, at
// the exact agent.Subagent construction Review performs, that only a
// prompt string reaches the model. This exercises the same Subagent{}
// shape subagentReviewer.Review builds (provider, model, system prompt,
// reviewerToolSet, MaxSteps) directly against a fake chat.Provider, so it
// proves the construction fact without needing a full *runtime.Runtime:
// Subagent.Run populates GenerateOptions.Prompt and never Messages, so
// there is no code path for the implementer's transcript to travel
// through.
func TestReviewSubagentConstructionNeverSendsConversationHistory(t *testing.T) {
captured := &capturingChatProvider{}
var findings []workflow.ReviewFinding
var checks []workflow.ReviewCheck
req := workflow.ReviewRequest{
SnapshotDir: t.TempDir(),
Diff: "diff --git a/x.go b/x.go\n",
IssueText: "some issue text",
}
sub := agent.Subagent{
Provider: captured,
Model: "fake-model",
System: reviewerSystemPrompt,
Tools: reviewerToolSet(req.SnapshotDir, &findings, &checks),
MaxSteps: defaultReviewMaxSteps,
}
prompt := reviewerPrompt(req)
if _, err := sub.Run(t.Context(), prompt); err != nil {
t.Fatalf("Subagent.Run() error = %v", err)
}
if captured.lastRequest == nil {
t.Fatal("chat provider was never called")
}
// The only non-system message the model ever sees is the one
// Subagent.Run derives from the prompt string -- there is no code
// path here for a second, implementer-authored message to arrive
// alongside it.
var userMessages int
for _, m := range captured.lastRequest.Messages {
if m.Role == chat.RoleUser {
userMessages++
if m.Content != prompt {
t.Errorf("user message content = %q, want the review prompt %q", m.Content, prompt)
}
}
}
if userMessages != 1 {
t.Errorf("chat.Request carried %d user messages, want exactly 1 (the review prompt, no implementer history)", userMessages)
}
}
func containsSubstr(s, substr string) bool {
return len(substr) == 0 ||
(len(s) >= len(substr) && func() bool {
for i := 0; i+len(substr) <= len(s); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}())
}
// capturingChatProvider is a minimal chat.Provider fake that records the
// last chat.Request it received and returns a fixed, valid completion.
type capturingChatProvider struct {
lastRequest *chat.Request
}
func (c *capturingChatProvider) Name() string { return "fake" }
func (c *capturingChatProvider) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
r := req
c.lastRequest = &r
return chat.Response{Content: "no findings", FinishReason: "stop"}, nil
}
func (c *capturingChatProvider) ChatStream(_ context.Context, req chat.Request) (chat.Stream, error) {
return nil, errors.New("not implemented")
}