Closes archie-core-h019.5. The shipped contract already made only a confirmed
error-level finding block, and kept zero-findings distinct from not-run; what
was missing was the other half of calibration, and that is what a reviewer that
cries wolf needs.
- ReviewReport gains Checked []ReviewCheck {Property, Evidence}: the properties
the reviewer verified clean. Evidence is required, for the same reason a
finding requires a failure scenario -- a check without evidence is an
assertion, not a verification.
- The reviewer records checks through a typed record_checked tool, mirroring
record_finding, so it cannot end its turn having silently skipped the
checklist.
- Style/formatting/naming nits are stated out of contract, in the ReviewFinding
type doc and the reviewer prompt: they are the padding that trains an
operator to skim.
- The PR-body findings section renders the cleared properties, so a
zero-finding review arrives backed by what was checked rather than reading
the same as a review that never looked.
Design: docs/prds/review-calibration.md.
253 lines
9.1 KiB
Go
253 lines
9.1 KiB
Go
package agentworker
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
|
|
"github.com/samcharles93/ai-sdk/agent"
|
|
"github.com/samcharles93/ai-sdk/chat"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
)
|
|
|
|
// TestReviewerToolSetIsReadOnlyAndHasNoMutationPath asserts, as a
|
|
// construction fact rather than a prompt claim, that the reviewer's
|
|
// toolset carries no tool capable of writing, editing, or running a
|
|
// shell command -- the class of tool that could reach the worktreerpc
|
|
// publication grant or otherwise mutate anything. Read/grep/find plus the
|
|
// findings-capture tool are the entire surface.
|
|
func TestReviewerToolSetIsReadOnlyAndHasNoMutationPath(t *testing.T) {
|
|
var findings []workflow.ReviewFinding
|
|
var checks []workflow.ReviewCheck
|
|
set := reviewerToolSet(t.TempDir(), &findings, &checks)
|
|
|
|
names := make([]string, 0, len(set))
|
|
for name := range set {
|
|
names = append(names, name)
|
|
}
|
|
slices.Sort(names)
|
|
|
|
want := []string{"find", "grep", "read", "record_checked", "record_finding"}
|
|
slices.Sort(want)
|
|
if !slices.Equal(names, want) {
|
|
t.Fatalf("reviewer toolset = %v, want exactly %v (no write/edit/shell)", names, want)
|
|
}
|
|
}
|
|
|
|
// TestReviewerToolSetIsRootedAtTheGivenSnapshot asserts the read tool
|
|
// cannot read outside the snapshot directory it was constructed with --
|
|
// it has no path back to the implementer's actual worktree.
|
|
func TestReviewerToolSetIsRootedAtTheGivenSnapshot(t *testing.T) {
|
|
snapshot := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(snapshot, "in-snapshot.txt"), []byte("ok\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
outside := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("nope\n"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var findings []workflow.ReviewFinding
|
|
var checks []workflow.ReviewCheck
|
|
set := reviewerToolSet(snapshot, &findings, &checks)
|
|
readTool := set["read"]
|
|
if readTool == nil {
|
|
t.Fatal("reviewer toolset has no read tool")
|
|
}
|
|
|
|
out, err := readTool.Execute(t.Context(), `{"path":"in-snapshot.txt"}`)
|
|
if err != nil {
|
|
t.Fatalf("read in-snapshot file: %v", err)
|
|
}
|
|
if !containsSubstr(out, "ok") {
|
|
t.Errorf("read in-snapshot file = %q, want it to contain the file's content", out)
|
|
}
|
|
|
|
// Tool failures are encoded in-band in the returned content, not as a
|
|
// Go error (toolkit.Registry.CoreToolSet's own contract) -- an escape
|
|
// attempt must reject there, not succeed.
|
|
escapeOut, err := readTool.Execute(t.Context(), `{"path":"`+filepath.Join(outside, "secret.txt")+`"}`)
|
|
if err != nil {
|
|
t.Fatalf("read outside-snapshot path: %v", err)
|
|
}
|
|
if containsSubstr(escapeOut, "nope") {
|
|
t.Errorf("read tool returned content from outside the snapshot: %q", escapeOut)
|
|
}
|
|
if !containsSubstr(escapeOut, "escapes") {
|
|
t.Errorf("read tool output = %q, want a path-confinement rejection", escapeOut)
|
|
}
|
|
}
|
|
|
|
// TestRecordFindingToolAppendsToCapturedSlice asserts the capture-tool
|
|
// contract: findings are recorded via the closure over the findings
|
|
// slice, not returned from Subagent.Run's text.
|
|
func TestRecordFindingToolAppendsToCapturedSlice(t *testing.T) {
|
|
var findings []workflow.ReviewFinding
|
|
tool := recordFindingTool(&findings)
|
|
|
|
input := `{"file":"main.go","line":10,"defect":"nil deref","failure_scenario":"calling Foo(nil) panics","verdict":"confirmed","level":"error","category":"nil-risk"}`
|
|
if _, err := tool.Execute(t.Context(), input); err != nil {
|
|
t.Fatalf("record_finding: %v", err)
|
|
}
|
|
if len(findings) != 1 {
|
|
t.Fatalf("captured %d findings, want 1", len(findings))
|
|
}
|
|
if !findings[0].Blocking() {
|
|
t.Error("captured finding is not blocking, want confirmed+error to block")
|
|
}
|
|
}
|
|
|
|
// TestRecordFindingToolRejectsInvalidVerdictWithoutError asserts an
|
|
// invalid finding is rejected in-band (so the model can retry) rather
|
|
// than surfacing as a hard tool error, and is not captured.
|
|
func TestRecordFindingToolRejectsInvalidVerdictWithoutError(t *testing.T) {
|
|
var findings []workflow.ReviewFinding
|
|
tool := recordFindingTool(&findings)
|
|
|
|
input := `{"file":"main.go","line":1,"defect":"x","failure_scenario":"y","verdict":"maybe","level":"error","category":"other"}`
|
|
out, err := tool.Execute(t.Context(), input)
|
|
if err != nil {
|
|
t.Fatalf("record_finding returned a Go error for invalid input: %v", err)
|
|
}
|
|
if !containsSubstr(out, "rejected") {
|
|
t.Errorf("record_finding output = %q, want a rejection message", out)
|
|
}
|
|
if len(findings) != 0 {
|
|
t.Errorf("captured %d findings, want 0 for an invalid finding", len(findings))
|
|
}
|
|
}
|
|
|
|
// TestRecordCheckedToolAppendsToCapturedSlice asserts the calibration capture
|
|
// contract: a cleared property is recorded through the closure, so a
|
|
// zero-finding review can report what it actually checked.
|
|
func TestRecordCheckedToolAppendsToCapturedSlice(t *testing.T) {
|
|
var checks []workflow.ReviewCheck
|
|
tool := recordCheckedTool(&checks)
|
|
|
|
input := `{"property":"nil-safety of Foo callers","evidence":"read all 4 callers; each nil-checks"}`
|
|
if _, err := tool.Execute(t.Context(), input); err != nil {
|
|
t.Fatalf("record_checked: %v", err)
|
|
}
|
|
if len(checks) != 1 || checks[0].Property != "nil-safety of Foo callers" {
|
|
t.Fatalf("captured checks = %+v, want one check for the recorded property", checks)
|
|
}
|
|
}
|
|
|
|
// TestRecordCheckedToolRejectsMissingEvidence asserts a check without evidence
|
|
// is rejected in-band and not captured, so "verified clean" cannot be recorded
|
|
// as a bare assertion.
|
|
func TestRecordCheckedToolRejectsMissingEvidence(t *testing.T) {
|
|
var checks []workflow.ReviewCheck
|
|
tool := recordCheckedTool(&checks)
|
|
|
|
out, err := tool.Execute(t.Context(), `{"property":"looks fine","evidence":""}`)
|
|
if err != nil {
|
|
t.Fatalf("record_checked returned a Go error for invalid input: %v", err)
|
|
}
|
|
if !containsSubstr(out, "rejected") {
|
|
t.Errorf("record_checked output = %q, want a rejection message", out)
|
|
}
|
|
if len(checks) != 0 {
|
|
t.Errorf("captured %d checks, want 0 for a check with no evidence", len(checks))
|
|
}
|
|
}
|
|
|
|
// TestCompletedReviewCarriesChecks asserts the report the reviewer returns
|
|
// carries the captured checks, not just the findings.
|
|
func TestCompletedReviewCarriesChecks(t *testing.T) {
|
|
checks := []workflow.ReviewCheck{{Property: "p", Evidence: "e"}}
|
|
report := completedReview(nil, checks, "summary")
|
|
if !report.Ran() {
|
|
t.Fatal("completedReview report did not run")
|
|
}
|
|
if len(report.Checked) != 1 || report.Checked[0].Property != "p" {
|
|
t.Fatalf("report.Checked = %+v, want the captured check", report.Checked)
|
|
}
|
|
}
|
|
|
|
// TestReviewSubagentConstructionNeverSendsConversationHistory asserts, at
|
|
// the exact agent.Subagent construction Review performs, that only a
|
|
// prompt string reaches the model. This exercises the same Subagent{}
|
|
// shape subagentReviewer.Review builds (provider, model, system prompt,
|
|
// reviewerToolSet, MaxSteps) directly against a fake chat.Provider, so it
|
|
// proves the construction fact without needing a full *runtime.Runtime:
|
|
// Subagent.Run populates GenerateOptions.Prompt and never Messages, so
|
|
// there is no code path for the implementer's transcript to travel
|
|
// through.
|
|
func TestReviewSubagentConstructionNeverSendsConversationHistory(t *testing.T) {
|
|
captured := &capturingChatProvider{}
|
|
var findings []workflow.ReviewFinding
|
|
var checks []workflow.ReviewCheck
|
|
req := workflow.ReviewRequest{
|
|
SnapshotDir: t.TempDir(),
|
|
Diff: "diff --git a/x.go b/x.go\n",
|
|
IssueText: "some issue text",
|
|
}
|
|
|
|
sub := agent.Subagent{
|
|
Provider: captured,
|
|
Model: "fake-model",
|
|
System: reviewerSystemPrompt,
|
|
Tools: reviewerToolSet(req.SnapshotDir, &findings, &checks),
|
|
MaxSteps: defaultReviewMaxSteps,
|
|
}
|
|
prompt := reviewerPrompt(req)
|
|
if _, err := sub.Run(t.Context(), prompt); err != nil {
|
|
t.Fatalf("Subagent.Run() error = %v", err)
|
|
}
|
|
|
|
if captured.lastRequest == nil {
|
|
t.Fatal("chat provider was never called")
|
|
}
|
|
// The only non-system message the model ever sees is the one
|
|
// Subagent.Run derives from the prompt string -- there is no code
|
|
// path here for a second, implementer-authored message to arrive
|
|
// alongside it.
|
|
var userMessages int
|
|
for _, m := range captured.lastRequest.Messages {
|
|
if m.Role == chat.RoleUser {
|
|
userMessages++
|
|
if m.Content != prompt {
|
|
t.Errorf("user message content = %q, want the review prompt %q", m.Content, prompt)
|
|
}
|
|
}
|
|
}
|
|
if userMessages != 1 {
|
|
t.Errorf("chat.Request carried %d user messages, want exactly 1 (the review prompt, no implementer history)", userMessages)
|
|
}
|
|
}
|
|
|
|
func containsSubstr(s, substr string) bool {
|
|
return len(substr) == 0 ||
|
|
(len(s) >= len(substr) && func() bool {
|
|
for i := 0; i+len(substr) <= len(s); i++ {
|
|
if s[i:i+len(substr)] == substr {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}())
|
|
}
|
|
|
|
// capturingChatProvider is a minimal chat.Provider fake that records the
|
|
// last chat.Request it received and returns a fixed, valid completion.
|
|
type capturingChatProvider struct {
|
|
lastRequest *chat.Request
|
|
}
|
|
|
|
func (c *capturingChatProvider) Name() string { return "fake" }
|
|
|
|
func (c *capturingChatProvider) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
|
|
r := req
|
|
c.lastRequest = &r
|
|
return chat.Response{Content: "no findings", FinishReason: "stop"}, nil
|
|
}
|
|
|
|
func (c *capturingChatProvider) ChatStream(_ context.Context, req chat.Request) (chat.Stream, error) {
|
|
return nil, errors.New("not implemented")
|
|
}
|