From 37fce6e0b8b96d527dabd1b5406ac7bd4e034571 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 11 Sep 2026 20:18:06 +1000 Subject: [PATCH] feat(workflow): review calibration -- checked properties, style nits out of contract Closes archie-core-h019.5. The shipped contract already made only a confirmed error-level finding block, and kept zero-findings distinct from not-run; what was missing was the other half of calibration, and that is what a reviewer that cries wolf needs. - ReviewReport gains Checked []ReviewCheck {Property, Evidence}: the properties the reviewer verified clean. Evidence is required, for the same reason a finding requires a failure scenario -- a check without evidence is an assertion, not a verification. - The reviewer records checks through a typed record_checked tool, mirroring record_finding, so it cannot end its turn having silently skipped the checklist. - Style/formatting/naming nits are stated out of contract, in the ReviewFinding type doc and the reviewer prompt: they are the padding that trains an operator to skim. - The PR-body findings section renders the cleared properties, so a zero-finding review arrives backed by what was checked rather than reading the same as a review that never looked. Design: docs/prds/review-calibration.md. --- docs/prds/review-calibration.md | 76 +++++++++++++++++++ internal/app/agentworker/review.go | 51 +++++++++++-- internal/app/agentworker/review_test.go | 59 +++++++++++++- internal/domain/workflow/review.go | 9 +++ internal/domain/workflow/review_findings.go | 37 +++++++++ .../domain/workflow/review_findings_test.go | 36 +++++++++ .../domain/workflow/review_surface_test.go | 19 +++++ 7 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 docs/prds/review-calibration.md diff --git a/docs/prds/review-calibration.md b/docs/prds/review-calibration.md new file mode 100644 index 000000000..45c07ac72 --- /dev/null +++ b/docs/prds/review-calibration.md @@ -0,0 +1,76 @@ +# Review calibration -- decision + +**Status:** Proposed +**Beads issue:** `archie-core-h019.5` (parent `archie-core-h019`) + +Decides the half of calibration `docs/prds/adversarial-self-review.md` left +open: "the adversarial-confirmation mechanism that keeps a finding from +surfacing on a single reviewer's say-so." + +## The problem, precisely + +The shipped contract (`internal/domain/workflow/review_findings.go`, h019.2) +already makes only a **confirmed** error-level finding block +(`Blocking() == Verdict == confirmed && Level == error`), and a zero-finding +review is already distinguishable from one that never ran (`ReviewStatus`). +What is missing is the other half: a reviewer can currently report *nothing* +and say only "I found nothing" in free text. The operator cannot tell "I +checked the five things that matter and they are clean" from "I did not look." +That is the failure the epic names -- a findings surface that consumes attention +without earning trust. + +## Decisions + +### 1. "Checked and cleared" becomes structured, not prose + +`ReviewReport` gains `Checked []ReviewCheck`: + +```go +type ReviewCheck struct { + Property string // what was verified clean, e.g. "nil-safety of Foo callers" + Evidence string // how it was verified, e.g. "read all 4 callers; each nil-checks" +} +``` + +`Evidence` is required, for the same reason `ReviewFinding.FailureScenario` is: +a check without evidence is an assertion, and the whole point is to separate a +verified property from an unexamined one. A zero-finding review is credible +only when it lists what it checked. + +### 2. Report checks through a typed tool, like findings + +The reviewer calls a new capture tool `record_checked` (mirroring +`record_finding`) once per property it verified clean. The same environmental +argument as findings applies: the reviewer cannot end its turn having silently +skipped the checklist. `ReviewReport.Summary` stays as the human overview. + +### 3. Style nits are out of contract + +A finding must describe wrong behaviour or a crash. Style, formatting, naming, +and preference nits are explicitly out of contract -- they are the padding that +trains the operator to skim. Stated in the reviewer prompt and in the +`ReviewFinding` type's own documentation. Not machine-enforced: deciding "is +this a style nit" from free text is unreliable, and a wrong rejection is worse +than a tolerated one. The category enum already has no style bucket; the rule +closes the gap that would otherwise let one hide under `other`. + +### 4. Surfacing + +The PR-body findings section lists the cleared properties alongside findings, +so "ran and found nothing" arrives with its evidence. (Park reasons keep +rendering only blocking findings -- what a human needs to act on, not the full +checklist.) + +## What this does not decide + +- Adding a second, independent reviewer pass. The verdict distinction plus the + recorded checks is the confirmation mechanism; a second model pass is a + separate, heavier decision and nothing in `h019.5` requires it. +- How many findings can survive before `MaxSteps` is exhausted (a budget + question, already satisfied by the per-stage budget). + +## Packages this touches + +- `internal/domain/workflow` -- `ReviewCheck`, `ReviewReport.Checked`, validation. +- `internal/app/agentworker` -- `record_checked` tool, prompt rules. +- `internal/domain/workflow/review.go` -- render cleared properties on the PR body. diff --git a/internal/app/agentworker/review.go b/internal/app/agentworker/review.go index bc09d819a..dfc33edfd 100644 --- a/internal/app/agentworker/review.go +++ b/internal/app/agentworker/review.go @@ -101,7 +101,8 @@ func (s *subagentReviewer) Review(ctx context.Context, req workflow.ReviewReques } var findings []workflow.ReviewFinding - tools := reviewerToolSet(req.SnapshotDir, &findings) + var checks []workflow.ReviewCheck + tools := reviewerToolSet(req.SnapshotDir, &findings, &checks) maxSteps := req.MaxSteps if maxSteps <= 0 { @@ -128,9 +129,17 @@ func (s *subagentReviewer) Review(ctx context.Context, req workflow.ReviewReques if len(findings) == 0 { return workflow.NewNotRunReviewReport(runErr.Error()) } - return workflow.NewCompletedReviewReport(findings, "") + return completedReview(findings, checks, "") } - return workflow.NewCompletedReviewReport(findings, summary) + return completedReview(findings, checks, summary) +} + +// completedReview builds a completed report carrying the captured findings +// and the properties the reviewer verified clean. +func completedReview(findings []workflow.ReviewFinding, checks []workflow.ReviewCheck, summary string) workflow.ReviewReport { + report := workflow.NewCompletedReviewReport(findings, summary) + report.Checked = checks + return report } // reviewerSystemPrompt instructs the reviewer per CLAUDE.md's own @@ -145,6 +154,10 @@ Check for: dead code, unchecked errors, hardcoded values that should be paramete For every defect you find, call record_finding with: the file and line, a one-sentence defect statement, a concrete failure scenario (specific inputs or state that produce the wrong output or a crash -- not a hypothetical), a verdict, a level, and a category. +A finding must describe wrong behaviour or a crash. Style, formatting, naming, and preference nits are out of contract -- do not report them; they are exactly the padding that teaches an operator to skim past a findings list. + +As you clear each checklist property, call record_checked with the property and how you verified it (which files or call sites you read). Record every property you actually checked and found correct, even when you also found defects; a review that reports no findings must still record what it checked, because "I checked and it is fine" and "I did not look" must not read the same. + Verdict is "confirmed" only when you have traced the actual failure -- read the code paths involved and can state exactly how it goes wrong. Verdict is "plausible" for a real worry you have not fully traced. Only a "confirmed" finding at level "error" blocks the pull request; do not inflate a plausible worry to confirmed/error to make it count -- an unjustified block is worse than a missed one, because it teaches the operator to stop trusting your reports. Level is "error" for a defect that blocks; "warn" for something worth knowing but not blocking. @@ -168,11 +181,11 @@ func reviewerPrompt(req workflow.ReviewRequest) string { } // reviewerToolSet builds the reviewer's read-only toolset, rooted at -// snapshotDir, plus the structured findings-capture tool that appends -// directly to findings. This is a distinct toolset built fresh for each +// snapshotDir, plus the structured capture tools that append directly to +// findings and checks. This is a distinct toolset built fresh for each // review -- never the worker's own registry, which holds write/edit/shell // tools and the worktreerpc publication grant. -func reviewerToolSet(snapshotDir string, findings *[]workflow.ReviewFinding) core.ToolSet { +func reviewerToolSet(snapshotDir string, findings *[]workflow.ReviewFinding, checks *[]workflow.ReviewCheck) core.ToolSet { reg := toolkit.NewRegistry() readOnly := []toolkit.Tool{ toolkit.NewReadTool(snapshotDir, toolkit.NewReadTracker()), @@ -184,9 +197,35 @@ func reviewerToolSet(snapshotDir string, findings *[]workflow.ReviewFinding) cor } set := reg.CoreToolSet(toolkit.NonInteractiveBridge{}) set["record_finding"] = recordFindingTool(findings) + set["record_checked"] = recordCheckedTool(checks) return set } +// checkInput is the reviewer-facing shape of workflow.ReviewCheck. +type checkInput struct { + Property string `json:"property" jsonschema:"description=The property or code path you verified clean, e.g. 'nil-safety of Foo callers'."` + Evidence string `json:"evidence" jsonschema:"description=How you verified it, e.g. 'read all 4 callers; each nil-checks before use'."` +} + +// recordCheckedTool builds the typed capture tool the reviewer calls once per +// property it verified clean. Without it a zero-finding review reads the same +// as a review that never looked; with it, "ran and found nothing" arrives +// backed by what was actually checked. +func recordCheckedTool(checks *[]workflow.ReviewCheck) *core.Tool { + return core.NewTypedTool( + "record_checked", + "Record one property or code path you verified clean. Call this once per property you actually checked and found correct -- a review that reports no findings must still record what it checked.", + func(_ context.Context, in checkInput) (string, error) { + check := workflow.ReviewCheck{Property: in.Property, Evidence: in.Evidence} + if err := check.Validate(); err != nil { + return "record_checked rejected: " + err.Error(), nil //nolint:nilerr // rejection feedback lets the model retry with a valid check + } + *checks = append(*checks, check) + return "check recorded", nil + }, + ) +} + // findingInput is the reviewer-facing shape of workflow.ReviewFinding: the // model reports plain strings for Verdict/Level/Category, validated and // converted before being appended. diff --git a/internal/app/agentworker/review_test.go b/internal/app/agentworker/review_test.go index cb1cff1bd..04d67ad4e 100644 --- a/internal/app/agentworker/review_test.go +++ b/internal/app/agentworker/review_test.go @@ -22,7 +22,8 @@ import ( // findings-capture tool are the entire surface. func TestReviewerToolSetIsReadOnlyAndHasNoMutationPath(t *testing.T) { var findings []workflow.ReviewFinding - set := reviewerToolSet(t.TempDir(), &findings) + var checks []workflow.ReviewCheck + set := reviewerToolSet(t.TempDir(), &findings, &checks) names := make([]string, 0, len(set)) for name := range set { @@ -30,7 +31,7 @@ func TestReviewerToolSetIsReadOnlyAndHasNoMutationPath(t *testing.T) { } slices.Sort(names) - want := []string{"find", "grep", "read", "record_finding"} + want := []string{"find", "grep", "read", "record_checked", "record_finding"} slices.Sort(want) if !slices.Equal(names, want) { t.Fatalf("reviewer toolset = %v, want exactly %v (no write/edit/shell)", names, want) @@ -51,7 +52,8 @@ func TestReviewerToolSetIsRootedAtTheGivenSnapshot(t *testing.T) { } var findings []workflow.ReviewFinding - set := reviewerToolSet(snapshot, &findings) + var checks []workflow.ReviewCheck + set := reviewerToolSet(snapshot, &findings, &checks) readTool := set["read"] if readTool == nil { t.Fatal("reviewer toolset has no read tool") @@ -119,6 +121,54 @@ func TestRecordFindingToolRejectsInvalidVerdictWithoutError(t *testing.T) { } } +// TestRecordCheckedToolAppendsToCapturedSlice asserts the calibration capture +// contract: a cleared property is recorded through the closure, so a +// zero-finding review can report what it actually checked. +func TestRecordCheckedToolAppendsToCapturedSlice(t *testing.T) { + var checks []workflow.ReviewCheck + tool := recordCheckedTool(&checks) + + input := `{"property":"nil-safety of Foo callers","evidence":"read all 4 callers; each nil-checks"}` + if _, err := tool.Execute(t.Context(), input); err != nil { + t.Fatalf("record_checked: %v", err) + } + if len(checks) != 1 || checks[0].Property != "nil-safety of Foo callers" { + t.Fatalf("captured checks = %+v, want one check for the recorded property", checks) + } +} + +// TestRecordCheckedToolRejectsMissingEvidence asserts a check without evidence +// is rejected in-band and not captured, so "verified clean" cannot be recorded +// as a bare assertion. +func TestRecordCheckedToolRejectsMissingEvidence(t *testing.T) { + var checks []workflow.ReviewCheck + tool := recordCheckedTool(&checks) + + out, err := tool.Execute(t.Context(), `{"property":"looks fine","evidence":""}`) + if err != nil { + t.Fatalf("record_checked returned a Go error for invalid input: %v", err) + } + if !containsSubstr(out, "rejected") { + t.Errorf("record_checked output = %q, want a rejection message", out) + } + if len(checks) != 0 { + t.Errorf("captured %d checks, want 0 for a check with no evidence", len(checks)) + } +} + +// TestCompletedReviewCarriesChecks asserts the report the reviewer returns +// carries the captured checks, not just the findings. +func TestCompletedReviewCarriesChecks(t *testing.T) { + checks := []workflow.ReviewCheck{{Property: "p", Evidence: "e"}} + report := completedReview(nil, checks, "summary") + if !report.Ran() { + t.Fatal("completedReview report did not run") + } + if len(report.Checked) != 1 || report.Checked[0].Property != "p" { + t.Fatalf("report.Checked = %+v, want the captured check", report.Checked) + } +} + // TestReviewSubagentConstructionNeverSendsConversationHistory asserts, at // the exact agent.Subagent construction Review performs, that only a // prompt string reaches the model. This exercises the same Subagent{} @@ -131,6 +181,7 @@ func TestRecordFindingToolRejectsInvalidVerdictWithoutError(t *testing.T) { func TestReviewSubagentConstructionNeverSendsConversationHistory(t *testing.T) { captured := &capturingChatProvider{} var findings []workflow.ReviewFinding + var checks []workflow.ReviewCheck req := workflow.ReviewRequest{ SnapshotDir: t.TempDir(), Diff: "diff --git a/x.go b/x.go\n", @@ -141,7 +192,7 @@ func TestReviewSubagentConstructionNeverSendsConversationHistory(t *testing.T) { Provider: captured, Model: "fake-model", System: reviewerSystemPrompt, - Tools: reviewerToolSet(req.SnapshotDir, &findings), + Tools: reviewerToolSet(req.SnapshotDir, &findings, &checks), MaxSteps: defaultReviewMaxSteps, } prompt := reviewerPrompt(req) diff --git a/internal/domain/workflow/review.go b/internal/domain/workflow/review.go index 1f27ba109..1824e0564 100644 --- a/internal/domain/workflow/review.go +++ b/internal/domain/workflow/review.go @@ -153,6 +153,15 @@ func renderPRReviewSection(report ReviewReport) string { b.WriteString("\n") } } + // The cleared properties are what back "ran and found nothing": without + // them a zero-finding review reads the same as one that never looked + // (h019.5 calibration). + if len(report.Checked) > 0 { + b.WriteString("\nChecked and cleared:\n\n") + for _, c := range report.Checked { + fmt.Fprintf(&b, "- **%s** — %s\n", c.Property, c.Evidence) + } + } if report.Summary != "" { fmt.Fprintf(&b, "\nSummary: %s\n", report.Summary) } diff --git a/internal/domain/workflow/review_findings.go b/internal/domain/workflow/review_findings.go index 1130149dc..75a55d483 100644 --- a/internal/domain/workflow/review_findings.go +++ b/internal/domain/workflow/review_findings.go @@ -75,6 +75,10 @@ const ( // ReviewFinding represents one defect found during adversarial review. // It is serialisable across the agent-execution JSON data boundary. +// +// A finding describes wrong behaviour or a crash. Style, formatting, naming, +// and preference nits are out of contract -- they are the padding that trains +// an operator to skim past a findings list. type ReviewFinding struct { // File is the repo-relative file path where the defect is located. File string `json:"file"` @@ -94,6 +98,18 @@ type ReviewFinding struct { Category ReviewCategory `json:"category"` } +// ReviewCheck records a property the reviewer verified clean. It is the +// symmetric counterpart of ReviewFinding: where a finding carries a failure +// scenario as its evidence, a check carries how the property was verified. A +// zero-finding review is credible only when it lists what it checked, so a +// check without evidence is an assertion, not a verification. +type ReviewCheck struct { + // Property is what was verified clean, e.g. "nil-safety of Foo callers". + Property string `json:"property"` + // Evidence is how it was verified, e.g. "read all 4 callers; each nil-checks". + Evidence string `json:"evidence"` +} + // ReviewReport represents the aggregate result of an adversarial review. // It explicitly distinguishes "reviewer looked and found nothing" (Status == ReviewStatusCompleted, Findings empty) // from "reviewer never ran" (Status == ReviewStatusNotRun or uninitialised). @@ -102,6 +118,10 @@ type ReviewReport struct { Status ReviewStatus `json:"status"` // Findings contains all findings produced by the review. Findings []ReviewFinding `json:"findings,omitempty"` + // Checked records the properties the reviewer verified clean. A + // zero-finding review is credible only when it says what it checked; + // this is what separates "I checked and it is fine" from "I did not look". + Checked []ReviewCheck `json:"checked,omitempty"` // Summary is a human-readable high-level overview of the review. Summary string `json:"summary,omitempty"` // SkipReason explains why the review was not run or was skipped. @@ -194,6 +214,18 @@ func (f ReviewFinding) Validate() error { return nil } +// Validate checks that the check satisfies contract invariants. Evidence is +// required: a check without evidence is an assertion, not a verification. +func (c ReviewCheck) Validate() error { + if c.Property == "" { + return errors.New("review check property is required") + } + if c.Evidence == "" { + return errors.New("review check evidence is required") + } + return nil +} + // Validate checks that the report satisfies contract invariants. func (r ReviewReport) Validate() error { switch r.Status { @@ -206,6 +238,11 @@ func (r ReviewReport) Validate() error { return fmt.Errorf("findings[%d]: %w", i, err) } } + for i, c := range r.Checked { + if err := c.Validate(); err != nil { + return fmt.Errorf("checked[%d]: %w", i, err) + } + } return nil } diff --git a/internal/domain/workflow/review_findings_test.go b/internal/domain/workflow/review_findings_test.go index cf677040e..8e2f96d5e 100644 --- a/internal/domain/workflow/review_findings_test.go +++ b/internal/domain/workflow/review_findings_test.go @@ -589,6 +589,18 @@ func TestReviewReportJSONRoundTrip(t *testing.T) { Summary: "Clean review pass", }, }, + { + name: "zero findings with checked properties", + report: ReviewReport{ + Status: ReviewStatusCompleted, + Findings: []ReviewFinding{}, + Checked: []ReviewCheck{ + {Property: "nil-safety of Foo callers", Evidence: "read all 4 callers; each nil-checks"}, + {Property: "error propagation in Bar", Evidence: "traced Bar through to the caller's error return"}, + }, + Summary: "Checked and cleared", + }, + }, { name: "not run report", report: ReviewReport{ @@ -874,6 +886,30 @@ func TestReviewReportValidation(t *testing.T) { }, wantError: true, }, + { + name: "valid completed report with checked properties", + report: ReviewReport{ + Status: ReviewStatusCompleted, + Checked: []ReviewCheck{{Property: "nil-safety of Foo callers", Evidence: "read all 4 callers"}}, + }, + wantError: false, + }, + { + name: "checked entry without evidence is rejected", + report: ReviewReport{ + Status: ReviewStatusCompleted, + Checked: []ReviewCheck{{Property: "looks fine"}}, + }, + wantError: true, + }, + { + name: "checked entry without property is rejected", + report: ReviewReport{ + Status: ReviewStatusCompleted, + Checked: []ReviewCheck{{Evidence: "read the file"}}, + }, + wantError: true, + }, { name: "invalid finding inside findings slice", report: ReviewReport{ diff --git a/internal/domain/workflow/review_surface_test.go b/internal/domain/workflow/review_surface_test.go index f4ab109a3..c601e1f78 100644 --- a/internal/domain/workflow/review_surface_test.go +++ b/internal/domain/workflow/review_surface_test.go @@ -27,6 +27,25 @@ func TestRenderPRReviewSection(t *testing.T) { report: NewCompletedReviewReport(nil, "checked and cleared"), want: []string{"Adversarial self-review ran", "found no defects"}, }, + { + name: "checked properties back a zero-finding review", + report: ReviewReport{ + Status: ReviewStatusCompleted, + Checked: []ReviewCheck{{Property: "nil-safety of Foo callers", Evidence: "read all 4 callers"}}, + }, + want: []string{"found no defects", "Checked and cleared", "nil-safety of Foo callers", "read all 4 callers"}, + }, + { + name: "checked properties also render alongside findings", + report: ReviewReport{ + Status: ReviewStatusCompleted, + Findings: []ReviewFinding{ + {File: "a.go", Line: 1, Defect: "warn", FailureScenario: "x", Verdict: ReviewVerdictPlausible, Level: ReviewLevelWarn, Category: ReviewCategoryOther}, + }, + Checked: []ReviewCheck{{Property: "error paths", Evidence: "traced two"}}, + }, + want: []string{"Non-blocking notes", "Checked and cleared", "error paths"}, + }, { name: "non-blocking findings listed", report: NewCompletedReviewReport([]ReviewFinding{