Closes archie-core-h019.5. The shipped contract already made only a confirmed
error-level finding block, and kept zero-findings distinct from not-run; what
was missing was the other half of calibration, and that is what a reviewer that
cries wolf needs.
- ReviewReport gains Checked []ReviewCheck {Property, Evidence}: the properties
the reviewer verified clean. Evidence is required, for the same reason a
finding requires a failure scenario -- a check without evidence is an
assertion, not a verification.
- The reviewer records checks through a typed record_checked tool, mirroring
record_finding, so it cannot end its turn having silently skipped the
checklist.
- Style/formatting/naming nits are stated out of contract, in the ReviewFinding
type doc and the reviewer prompt: they are the padding that trains an
operator to skim.
- The PR-body findings section renders the cleared properties, so a
zero-finding review arrives backed by what was checked rather than reading
the same as a review that never looked.
Design: docs/prds/review-calibration.md.
269 lines
13 KiB
Go
269 lines
13 KiB
Go
package agentworker
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/samcharles93/ai-sdk/agent"
|
|
"github.com/samcharles93/ai-sdk/core"
|
|
"github.com/samcharles93/ai-sdk/runtime"
|
|
"github.com/samcharles93/ai-sdk/toolkit"
|
|
|
|
"github.com/samcharles93/archie-core/internal/agentexec"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
"github.com/samcharles93/archie-core/internal/taskrun"
|
|
)
|
|
|
|
// newReviewerFor builds the workflow.Reviewer for one task run, or nil
|
|
// when the repo has not opted into review (Repo.ReviewEnabled == false)
|
|
// or no providers are configured. Model resolution mirrors AgentStage's
|
|
// own role->"builder" fallback (agent.go): Models["reviewer"], falling
|
|
// back to Models["builder"].
|
|
func newReviewerFor(req taskrun.Request) workflow.Reviewer {
|
|
if !req.Repo.ReviewEnabled {
|
|
return nil
|
|
}
|
|
return NewOperatorReviewer(req.Cfg.Models, req.Providers)
|
|
}
|
|
|
|
// NewOperatorReviewer builds the workflow.Reviewer for an operator-triggered
|
|
// review, resolved from daemon-level models and providers rather than a task
|
|
// run. It shares newReviewerFor's role->"builder" model fallback but drops
|
|
// the Repo.ReviewEnabled gate: the operator asked for the review explicitly,
|
|
// so "not opted in" is not a reason to refuse. It returns nil when no model
|
|
// resolves (no reviewer/builder model configured) or no provider runtime
|
|
// builds, so the caller can report a clear "review not configured" result.
|
|
func NewOperatorReviewer(models map[string]string, providers map[string]agentexec.Provider) workflow.Reviewer {
|
|
modelRef := ReviewerModel(models)
|
|
if modelRef == "" {
|
|
return nil
|
|
}
|
|
rt := agentexec.NewRuntime(providers)
|
|
if rt == nil {
|
|
return nil
|
|
}
|
|
return newSubagentReviewer(rt, modelRef)
|
|
}
|
|
|
|
// ReviewerModel resolves the reviewer role's model with the builder fallback.
|
|
// It is the single source of truth for the role->"builder" rule, shared by
|
|
// the workflow stage (via NewOperatorReviewer/newReviewerFor) and the
|
|
// operator-triggered review's result reporting so the reported model and the
|
|
// model actually run cannot diverge.
|
|
func ReviewerModel(models map[string]string) string {
|
|
if models["reviewer"] != "" {
|
|
return models["reviewer"]
|
|
}
|
|
return models["builder"]
|
|
}
|
|
|
|
// defaultReviewMaxSteps bounds the reviewer's tool loop when neither
|
|
// ReviewRequest.MaxSteps nor the repo's configured budget set one.
|
|
const defaultReviewMaxSteps = 10
|
|
|
|
// subagentReviewer implements workflow.Reviewer using ai-sdk's
|
|
// agent.Subagent: a nested, synchronous generation in this same worker
|
|
// process, with its own model, system prompt, toolset, and step budget.
|
|
//
|
|
// Isolation is two-fold. Conversation isolation is structural: Subagent.Run
|
|
// only ever sends a prompt string, never the implementer's message
|
|
// history. Workspace isolation is built by the caller (StageReview via
|
|
// workflow.Trees.Snapshot): reviewerToolSet is rooted at that snapshot
|
|
// directory and holds only read-only tools -- never the worker's own
|
|
// toolset, which carries the worktreerpc publication grant a reviewer must
|
|
// not be able to reach.
|
|
type subagentReviewer struct {
|
|
runtime *runtime.Runtime
|
|
modelRef string
|
|
}
|
|
|
|
// newSubagentReviewer constructs a Reviewer that runs on modelRef, resolved
|
|
// through rt (the same provider runtime the worker's own agent runner
|
|
// uses). rt must be non-nil.
|
|
func newSubagentReviewer(rt *runtime.Runtime, modelRef string) *subagentReviewer {
|
|
return &subagentReviewer{runtime: rt, modelRef: modelRef}
|
|
}
|
|
|
|
var _ workflow.Reviewer = (*subagentReviewer)(nil)
|
|
|
|
// Review runs one adversarial review. It never returns a Go error: every
|
|
// failure mode (unresolvable model, a Subagent.Run error, a truncated
|
|
// run) folds into the returned ReviewReport, per
|
|
// docs/prds/adversarial-self-review.md section 4's fail-closed contract.
|
|
func (s *subagentReviewer) Review(ctx context.Context, req workflow.ReviewRequest) workflow.ReviewReport {
|
|
if s.runtime == nil {
|
|
return workflow.NewNotRunReviewReport("no provider runtime configured for the reviewer")
|
|
}
|
|
provider, model, err := s.runtime.ChatProvider(ctx, s.modelRef)
|
|
if err != nil {
|
|
return workflow.NewNotRunReviewReport(fmt.Sprintf("resolve reviewer model %q: %v", s.modelRef, err))
|
|
}
|
|
|
|
var findings []workflow.ReviewFinding
|
|
var checks []workflow.ReviewCheck
|
|
tools := reviewerToolSet(req.SnapshotDir, &findings, &checks)
|
|
|
|
maxSteps := req.MaxSteps
|
|
if maxSteps <= 0 {
|
|
maxSteps = defaultReviewMaxSteps
|
|
}
|
|
|
|
sub := agent.Subagent{
|
|
Provider: provider,
|
|
Model: model,
|
|
System: reviewerSystemPrompt,
|
|
Tools: tools,
|
|
MaxSteps: maxSteps,
|
|
}
|
|
|
|
summary, runErr := sub.Run(ctx, reviewerPrompt(req))
|
|
if runErr != nil {
|
|
// A truncated or errored run still yields whatever the reviewer
|
|
// captured before it ran out of budget -- those findings are real
|
|
// and must not be discarded just because the run's own
|
|
// conclusion never arrived (docs/prds/adversarial-self-review.md
|
|
// section 1, "findings return through a closure"). Zero captured
|
|
// findings on error means the reviewer never got anywhere, which
|
|
// fail-closed treats as not having run at all.
|
|
if len(findings) == 0 {
|
|
return workflow.NewNotRunReviewReport(runErr.Error())
|
|
}
|
|
return completedReview(findings, checks, "")
|
|
}
|
|
return completedReview(findings, checks, summary)
|
|
}
|
|
|
|
// completedReview builds a completed report carrying the captured findings
|
|
// and the properties the reviewer verified clean.
|
|
func completedReview(findings []workflow.ReviewFinding, checks []workflow.ReviewCheck, summary string) workflow.ReviewReport {
|
|
report := workflow.NewCompletedReviewReport(findings, summary)
|
|
report.Checked = checks
|
|
return report
|
|
}
|
|
|
|
// reviewerSystemPrompt instructs the reviewer per CLAUDE.md's own
|
|
// adversarial-review discipline: assume every line is wrong until proven
|
|
// correct, work the checklist, report only what a fresh reader can verify
|
|
// from what it was given.
|
|
const reviewerSystemPrompt = `You are a fresh adversarial code reviewer. You did not write this change and have no access to how or why it was written -- only what is given to you below.
|
|
|
|
Assume every line is wrong until you have verified it is correct. Read whole files with your read tool, not just diff hunks, before concluding a call site or interface is satisfied. Use grep/find to check for other call sites a change affects.
|
|
|
|
Check for: dead code, unchecked errors, hardcoded values that should be parameters, interface-satisfaction defects, nil-pointer risk, goroutine leaks, races and unsynchronized shared state.
|
|
|
|
For every defect you find, call record_finding with: the file and line, a one-sentence defect statement, a concrete failure scenario (specific inputs or state that produce the wrong output or a crash -- not a hypothetical), a verdict, a level, and a category.
|
|
|
|
A finding must describe wrong behaviour or a crash. Style, formatting, naming, and preference nits are out of contract -- do not report them; they are exactly the padding that teaches an operator to skim past a findings list.
|
|
|
|
As you clear each checklist property, call record_checked with the property and how you verified it (which files or call sites you read). Record every property you actually checked and found correct, even when you also found defects; a review that reports no findings must still record what it checked, because "I checked and it is fine" and "I did not look" must not read the same.
|
|
|
|
Verdict is "confirmed" only when you have traced the actual failure -- read the code paths involved and can state exactly how it goes wrong. Verdict is "plausible" for a real worry you have not fully traced. Only a "confirmed" finding at level "error" blocks the pull request; do not inflate a plausible worry to confirmed/error to make it count -- an unjustified block is worse than a missed one, because it teaches the operator to stop trusting your reports.
|
|
|
|
Level is "error" for a defect that blocks; "warn" for something worth knowing but not blocking.
|
|
|
|
When you have finished checking, reply with a short summary of what you checked and what you found (or that you found nothing). If you find nothing, say so plainly -- reporting zero findings on properties you actually checked is a valid and expected outcome, not a failure to find something.`
|
|
|
|
// reviewerPrompt renders the reviewer's mission: the diff and issue text.
|
|
// The snapshot itself is not embedded here -- it reaches the reviewer only
|
|
// through its read-only toolset, rooted at SnapshotDir, never as prompt
|
|
// text the reviewer could mistake for something it already checked.
|
|
func reviewerPrompt(req workflow.ReviewRequest) string {
|
|
var b strings.Builder
|
|
b.WriteString("Review the following change. The full file tree at the reviewed commit is available through your tools; ")
|
|
b.WriteString("start by reading the files the diff touches, not just the diff itself.\n\n")
|
|
b.WriteString("<issue>\n")
|
|
b.WriteString(req.IssueText)
|
|
b.WriteString("\n</issue>\n\n<diff>\n")
|
|
b.WriteString(req.Diff)
|
|
b.WriteString("\n</diff>\n")
|
|
return b.String()
|
|
}
|
|
|
|
// reviewerToolSet builds the reviewer's read-only toolset, rooted at
|
|
// snapshotDir, plus the structured capture tools that append directly to
|
|
// findings and checks. This is a distinct toolset built fresh for each
|
|
// review -- never the worker's own registry, which holds write/edit/shell
|
|
// tools and the worktreerpc publication grant.
|
|
func reviewerToolSet(snapshotDir string, findings *[]workflow.ReviewFinding, checks *[]workflow.ReviewCheck) core.ToolSet {
|
|
reg := toolkit.NewRegistry()
|
|
readOnly := []toolkit.Tool{
|
|
toolkit.NewReadTool(snapshotDir, toolkit.NewReadTracker()),
|
|
toolkit.NewGrepTool(snapshotDir),
|
|
toolkit.NewFindTool(snapshotDir),
|
|
}
|
|
for _, tool := range readOnly {
|
|
_ = reg.Register(tool) // names are fixed and distinct; Register cannot fail here
|
|
}
|
|
set := reg.CoreToolSet(toolkit.NonInteractiveBridge{})
|
|
set["record_finding"] = recordFindingTool(findings)
|
|
set["record_checked"] = recordCheckedTool(checks)
|
|
return set
|
|
}
|
|
|
|
// checkInput is the reviewer-facing shape of workflow.ReviewCheck.
|
|
type checkInput struct {
|
|
Property string `json:"property" jsonschema:"description=The property or code path you verified clean, e.g. 'nil-safety of Foo callers'."`
|
|
Evidence string `json:"evidence" jsonschema:"description=How you verified it, e.g. 'read all 4 callers; each nil-checks before use'."`
|
|
}
|
|
|
|
// recordCheckedTool builds the typed capture tool the reviewer calls once per
|
|
// property it verified clean. Without it a zero-finding review reads the same
|
|
// as a review that never looked; with it, "ran and found nothing" arrives
|
|
// backed by what was actually checked.
|
|
func recordCheckedTool(checks *[]workflow.ReviewCheck) *core.Tool {
|
|
return core.NewTypedTool(
|
|
"record_checked",
|
|
"Record one property or code path you verified clean. Call this once per property you actually checked and found correct -- a review that reports no findings must still record what it checked.",
|
|
func(_ context.Context, in checkInput) (string, error) {
|
|
check := workflow.ReviewCheck{Property: in.Property, Evidence: in.Evidence}
|
|
if err := check.Validate(); err != nil {
|
|
return "record_checked rejected: " + err.Error(), nil //nolint:nilerr // rejection feedback lets the model retry with a valid check
|
|
}
|
|
*checks = append(*checks, check)
|
|
return "check recorded", nil
|
|
},
|
|
)
|
|
}
|
|
|
|
// findingInput is the reviewer-facing shape of workflow.ReviewFinding: the
|
|
// model reports plain strings for Verdict/Level/Category, validated and
|
|
// converted before being appended.
|
|
type findingInput struct {
|
|
File string `json:"file" jsonschema:"description=Repo-relative file path where the defect is located."`
|
|
Line int `json:"line" jsonschema:"description=Line number where the defect is located."`
|
|
Defect string `json:"defect" jsonschema:"description=One-sentence statement of the defect."`
|
|
FailureScenario string `json:"failure_scenario" jsonschema:"description=Concrete inputs or state that produce the wrong output or a crash -- not a hypothetical."`
|
|
Verdict string `json:"verdict" jsonschema:"description=Only 'confirmed' may block. Use 'plausible' for a real worry you have not fully traced.,enum=confirmed|plausible"`
|
|
Level string `json:"level" jsonschema:"description=error blocks when confirmed; warn is advisory only.,enum=error|warn"`
|
|
Category string `json:"category" jsonschema:"enum=dead-code|unchecked-error|hardcoded-value|interface-satisfaction|nil-risk|goroutine-leak|race|other"`
|
|
}
|
|
|
|
// recordFindingTool builds the typed capture tool the reviewer must call
|
|
// to report a defect. Rejecting an invalid finding with a plain-text
|
|
// message (rather than a Go error) lets the model see what was wrong and
|
|
// retry, the same environmental-enforcement pattern the rest of the
|
|
// agent-loop tool contract uses.
|
|
func recordFindingTool(findings *[]workflow.ReviewFinding) *core.Tool {
|
|
return core.NewTypedTool(
|
|
"record_finding",
|
|
"Report one defect found during review. Call this once per defect; do not summarize multiple defects into one call.",
|
|
func(_ context.Context, in findingInput) (string, error) {
|
|
finding := workflow.ReviewFinding{
|
|
File: in.File,
|
|
Line: in.Line,
|
|
Defect: in.Defect,
|
|
FailureScenario: in.FailureScenario,
|
|
Verdict: workflow.ReviewVerdict(in.Verdict),
|
|
Level: workflow.ReviewLevel(in.Level),
|
|
Category: workflow.ReviewCategory(in.Category),
|
|
}
|
|
if err := finding.Validate(); err != nil {
|
|
return "record_finding rejected: " + err.Error(), nil //nolint:nilerr // rejection feedback lets the model retry with a valid finding
|
|
}
|
|
*findings = append(*findings, finding)
|
|
return "finding recorded", nil
|
|
},
|
|
)
|
|
}
|