feat(workflow): wire pr-review's verification and coverage/consistency stages
Adds phases 5 and 6 of docs/prds/pr-review-agent.md between the existing review and synthesis stages: an evidence verifier that drops unsupported high-priority findings, an adversary that confirms/challenges findings and may report new ones (more sceptical for a likely machine-written PR), a per-cluster compound-defect check, a coverage gate that runs gap reviewers for uncovered clusters and high-exposure blast-radius files (capped at 2 rounds), and consistency verification for cross-location obligations. The coverage gate and consistency verification run in parallel per the PRD, guarded by prReviewState.findingsMu. Phases 5/6 are afbk.4's scope; the merge gate (phase 8) remains afbk.5's.
This commit is contained in:
1 parent
89c5f8e2ad
commit
d4e254b349
3 files changed
+765
-41
No files matched your search
@@ -58,11 +58,23 @@ type prReviewState struct {
|
||||
snapshotDir string
|
||||
headSHA string
|
||||
blastRadius []string
|
||||
// clusters and highExposure are phase 6's coverage-gate inputs, computed
|
||||
// once in anatomy alongside blastRadius: clusters groups changed files by
|
||||
// directory, highExposure is the blastRadius files more than one changed
|
||||
// file reaches.
|
||||
clusters []prreview.Cluster
|
||||
highExposure []string
|
||||
|
||||
narrative string
|
||||
dimensions []prreview.Dimension
|
||||
findings []prreview.Finding
|
||||
scored []prreview.ScoredFinding
|
||||
|
||||
// findingsMu guards findings during phase 6, where the coverage gate and
|
||||
// consistency verification append to it from two goroutines running in
|
||||
// parallel (docs/prds/pr-review-agent.md, phase 6). Every other phase
|
||||
// touches findings from a single goroutine and does not need it.
|
||||
findingsMu sync.Mutex
|
||||
}
|
||||
|
||||
// prReviewConcurrency bounds phases 3 (lenses), 4 (reviewers) and 9 (polish),
|
||||
@@ -71,9 +83,9 @@ type prReviewState struct {
|
||||
const prReviewConcurrency = 8
|
||||
|
||||
// PRReview is the pull request reviewer: a fixed pipeline, not a single
|
||||
// agent (docs/prds/pr-review-agent.md). This bead wires phases 1, 2, 3, 4, 7
|
||||
// and 9; phases 5, 6 and 8 (verification, coverage/consistency, merge gate)
|
||||
// are archie-core-afbk.4 and .5.
|
||||
// agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2,
|
||||
// 3, 4, 7 and 9; archie-core-afbk.4 adds 5 and 6 (verification,
|
||||
// coverage/consistency); phase 8 (the merge gate) is archie-core-afbk.5.
|
||||
func PRReview() Workflow {
|
||||
return Workflow{
|
||||
Name: "pr-review",
|
||||
@@ -82,6 +94,8 @@ func PRReview() Workflow {
|
||||
stagePRAnatomy(),
|
||||
stagePRLenses(),
|
||||
stagePRReview(),
|
||||
stagePRVerification(),
|
||||
stagePRCoverageConsistency(),
|
||||
stagePRSynthesis(),
|
||||
stagePROutput(),
|
||||
},
|
||||
@@ -197,6 +211,12 @@ func stagePRAnatomy() Stage {
|
||||
return fmt.Errorf("compute blast radius: %w", err)
|
||||
}
|
||||
tc.prReview.blastRadius = blast
|
||||
tc.prReview.clusters = prreview.ClusterFiles(tc.prReview.files)
|
||||
exposure, err := prreview.ExposureCounts(os.DirFS(dir), changed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compute exposure counts: %w", err)
|
||||
}
|
||||
tc.prReview.highExposure = prreview.HighExposureFiles(exposure)
|
||||
|
||||
mission := fmt.Sprintf(
|
||||
"Read this pull request's diff and enough of the surrounding repository to "+
|
||||
@@ -349,31 +369,6 @@ func stagePRReview() Stage {
|
||||
// case a distinct StepFailed child step, so a reader of the execution tree
|
||||
// (not this slice) can tell "unreviewed" from "reviewed-clean" apart.
|
||||
func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) []prreview.Finding {
|
||||
params := json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": {"type": "string"},
|
||||
"line_start": {"type": "integer"},
|
||||
"line_end": {"type": "integer"},
|
||||
"severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]},
|
||||
"title": {"type": "string"},
|
||||
"body": {"type": "string"},
|
||||
"suggestion": {"type": "string"},
|
||||
"evidence": {"type": "string"},
|
||||
"confidence": {"type": "number"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}}
|
||||
},
|
||||
"required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["findings"]
|
||||
}`)
|
||||
mission := fmt.Sprintf(
|
||||
"%s\n\nTarget files: %s\n\nRead the target files (and, if useful, the context "+
|
||||
"files) in the snapshot and report every finding for this dimension. Quote the "+
|
||||
@@ -383,17 +378,63 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [
|
||||
dim.Prompt, strings.Join(dim.TargetFiles, ", "),
|
||||
)
|
||||
name := "reviewer-" + dim.Name
|
||||
res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{{
|
||||
Name: "report_findings", Description: "Record this dimension's findings. Call exactly once, before finish.",
|
||||
Parameters: params, RequiredFields: []string{"findings"}, MaxCalls: 1,
|
||||
}})
|
||||
res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{reportFindingsTool})
|
||||
if runErr != nil || res.Status != agentexec.StatusPassed {
|
||||
return nil
|
||||
}
|
||||
calls := res.Captures["report_findings"]
|
||||
if len(calls) != 1 {
|
||||
findings, err := decodeReportedFindings(res.Captures["report_findings"], dim.Name)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return findings
|
||||
}
|
||||
|
||||
// reportFindingsSchema is the finding shape every pr-review agent call that
|
||||
// reports findings captures them in: phase 4's reviewer, phase 6's gap
|
||||
// reviewers (which reuse runReviewer directly) and consistency verification.
|
||||
var reportFindingsSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": {"type": "string"},
|
||||
"line_start": {"type": "integer"},
|
||||
"line_end": {"type": "integer"},
|
||||
"severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]},
|
||||
"title": {"type": "string"},
|
||||
"body": {"type": "string"},
|
||||
"suggestion": {"type": "string"},
|
||||
"evidence": {"type": "string"},
|
||||
"confidence": {"type": "number"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}}
|
||||
},
|
||||
"required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["findings"]
|
||||
}`)
|
||||
|
||||
// reportFindingsTool is the capture tool every findings-reporting call
|
||||
// registers, at the shape reportFindingsSchema names.
|
||||
var reportFindingsTool = agentexec.CaptureTool{
|
||||
Name: "report_findings", Description: "Record findings. Call exactly once, before finish.",
|
||||
Parameters: reportFindingsSchema, RequiredFields: []string{"findings"}, MaxCalls: 1,
|
||||
}
|
||||
|
||||
// decodeReportedFindings decodes one report_findings capture into findings
|
||||
// tagged with dimension, shared by every call site reportFindingsSchema
|
||||
// backs. A call that never reported (no captures) or reported malformed JSON
|
||||
// decodes to no findings and an error the caller treats as "nothing to add",
|
||||
// not as a stage failure -- an agent that could not report is not evidence
|
||||
// the code that follows should stop.
|
||||
func decodeReportedFindings(calls []json.RawMessage, dimension string) ([]prreview.Finding, error) {
|
||||
if len(calls) != 1 {
|
||||
return nil, fmt.Errorf("report_findings called %d times (want exactly once)", len(calls))
|
||||
}
|
||||
var captured struct {
|
||||
Findings []struct {
|
||||
File string `json:"file"`
|
||||
@@ -409,7 +450,7 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [
|
||||
} `json:"findings"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return nil
|
||||
return nil, fmt.Errorf("decode findings: %w", err)
|
||||
}
|
||||
out := make([]prreview.Finding, len(captured.Findings))
|
||||
for i, f := range captured.Findings {
|
||||
@@ -418,12 +459,12 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [
|
||||
lineEnd = f.LineStart
|
||||
}
|
||||
out[i] = prreview.Finding{
|
||||
Dimension: dim.Name, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd,
|
||||
Dimension: dimension, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd,
|
||||
Severity: prreview.Severity(f.Severity), Title: f.Title, Body: f.Body,
|
||||
Suggestion: f.Suggestion, Evidence: f.Evidence, Confidence: f.Confidence, Tags: f.Tags,
|
||||
}
|
||||
}
|
||||
return out
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// prReviewMaxSteps bounds one reviewer's tool-loop iterations. Exhausting it
|
||||
@@ -475,10 +516,10 @@ func runReviewerAgent(
|
||||
|
||||
// stagePRSynthesis is pipeline phase 7, code only: score, drop findings
|
||||
// under their severity's confidence floor, merge duplicates, rank, and cap
|
||||
// at the inline comment limit. Phases 5, 6 and 8 (verification, coverage and
|
||||
// consistency, the merge gate) do not run yet, so every surviving finding is
|
||||
// scored on phase 4's raw output alone -- no compound, adversary or
|
||||
// merge-gate multiplier has anything to apply yet.
|
||||
// at the inline comment limit. Phase 8 (the merge gate) does not run yet, so
|
||||
// every surviving finding's Blocking flag is whatever phase 5/6 left it as
|
||||
// (false, since neither sets it -- that is archie-core-afbk.5's decision to
|
||||
// make).
|
||||
func stagePRSynthesis() Stage {
|
||||
return Stage{Name: "synthesis", Run: func(_ context.Context, tc *TaskContext) error {
|
||||
scored := prreview.Score(tc.prReview.findings, prreview.ScoreInputs{
|
||||
|
||||
@@ -0,0 +1,488 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
// stagePRVerification is pipeline phase 5, over the review findings phase 4
|
||||
// produced: extract each finding's evidence package (code), check the
|
||||
// high-priority findings against theirs (an evidence verifier agent), have
|
||||
// an adversary confirm or challenge every finding and add anything every
|
||||
// reviewer missed, then flag within-cluster compound defects
|
||||
// (docs/prds/pr-review-agent.md, phase 5).
|
||||
func stagePRVerification() Stage {
|
||||
return Stage{Name: "verification", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if len(tc.prReview.findings) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
verified, err := runEvidenceVerifier(ctx, tc, tc.prReview.findings)
|
||||
if err != nil {
|
||||
return fmt.Errorf("evidence verifier: %w", err)
|
||||
}
|
||||
|
||||
confirmed, err := runAdversary(ctx, tc, verified)
|
||||
if err != nil {
|
||||
return fmt.Errorf("adversary: %w", err)
|
||||
}
|
||||
|
||||
compounded, err := runCompoundCheck(ctx, tc, confirmed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compound check: %w", err)
|
||||
}
|
||||
|
||||
tc.prReview.findings = compounded
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// evidencePackageFor extracts a finding's evidence package from the
|
||||
// snapshot, reusing afbk.1's ExtractEvidence. A file the snapshot no longer
|
||||
// has (a finding pointing at a path that does not exist) extracts an empty
|
||||
// package rather than failing the stage: that emptiness is itself the
|
||||
// evidence verifier's answer.
|
||||
func evidencePackageFor(tc *TaskContext, f prreview.Finding) prreview.EvidencePackage {
|
||||
pkg, err := prreview.ExtractEvidence(os.DirFS(tc.prReview.snapshotDir), prreview.EvidenceRequest{
|
||||
File: f.File, LineStart: f.LineStart, LineEnd: f.LineEnd,
|
||||
Title: f.Title, Body: f.Body, Evidence: f.Evidence,
|
||||
})
|
||||
if err != nil {
|
||||
return prreview.EvidencePackage{FindingTitle: f.Title, File: f.File}
|
||||
}
|
||||
return pkg
|
||||
}
|
||||
|
||||
// verifyFindingsSchema is the evidence verifier's capture-tool schema.
|
||||
var verifyFindingsSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"verdicts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {"type": "integer"},
|
||||
"supported": {"type": "boolean"},
|
||||
"reason": {"type": "string"}
|
||||
},
|
||||
"required": ["index", "supported"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["verdicts"]
|
||||
}`)
|
||||
|
||||
// evidenceCandidate is one high-priority finding the evidence verifier is
|
||||
// asked to check, paired with its index in the caller's original slice: the
|
||||
// verifier's answer references findings by that index, not by content.
|
||||
type evidenceCandidate struct {
|
||||
index int
|
||||
finding prreview.Finding
|
||||
}
|
||||
|
||||
// highPriorityCandidates picks the findings runEvidenceVerifier checks:
|
||||
// every high-priority finding, each still carrying its position in findings.
|
||||
func highPriorityCandidates(findings []prreview.Finding) []evidenceCandidate {
|
||||
var candidates []evidenceCandidate
|
||||
for i, f := range findings {
|
||||
if prreview.IsHighPriority(f) {
|
||||
candidates = append(candidates, evidenceCandidate{index: i, finding: f})
|
||||
}
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
// evidenceListing renders each candidate's claim and evidence package as the
|
||||
// evidence verifier's mission text.
|
||||
func evidenceListing(tc *TaskContext, candidates []evidenceCandidate) string {
|
||||
var listing strings.Builder
|
||||
for _, c := range candidates {
|
||||
pkg := evidencePackageFor(tc, c.finding)
|
||||
fmt.Fprintf(&listing, "Finding %d: %s (%s:%d)\nClaim: %s\nQuoted evidence: %s\nCode at the cited lines:\n%s\n\n",
|
||||
c.index, c.finding.Title, c.finding.File, c.finding.LineStart, c.finding.Body, c.finding.Evidence, pkg.PrimaryCode)
|
||||
}
|
||||
return listing.String()
|
||||
}
|
||||
|
||||
// runEvidenceVerifier runs phase 5's evidence verifier over every
|
||||
// high-priority (critical or important) finding, dropping the ones it finds
|
||||
// unsupported by their own evidence package. A finding below high priority is
|
||||
// never sent, and always kept: the verifier exists to catch a hallucinated
|
||||
// critical, not to grade a nitpick. A failed or malformed verifier call keeps
|
||||
// every finding unfiltered -- the pipeline is recall-first, so a verifier
|
||||
// that could not run is not evidence any finding is wrong; the error is
|
||||
// intentionally swallowed here, not propagated.
|
||||
func runEvidenceVerifier(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
|
||||
candidates := highPriorityCandidates(findings)
|
||||
if len(candidates) == 0 {
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
mission := fmt.Sprintf(
|
||||
"Each finding below cites code as evidence for its claim. Check whether the "+
|
||||
"quoted code at the cited lines actually supports the claim -- not whether "+
|
||||
"the claim is a good catch, only whether the evidence is real and says what "+
|
||||
"the finding says it says.\n\n%s\n"+
|
||||
"Call verify_findings exactly once with one verdict per finding index, then "+
|
||||
"call finish with status \"passed\".",
|
||||
evidenceListing(tc, candidates),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "evidence-verifier", "review", mission, 15, []agentexec.CaptureTool{{
|
||||
Name: "verify_findings", Description: "Record each finding's evidence verdict. Call exactly once, before finish.",
|
||||
Parameters: verifyFindingsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return findings, nil //nolint:nilerr // recall-first: a verifier that could not run is not evidence any finding is wrong, so nothing is filtered
|
||||
}
|
||||
calls := res.Captures["verify_findings"]
|
||||
if len(calls) != 1 {
|
||||
return findings, nil
|
||||
}
|
||||
var captured struct {
|
||||
Verdicts []struct {
|
||||
Index int `json:"index"`
|
||||
Supported bool `json:"supported"`
|
||||
} `json:"verdicts"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return findings, nil //nolint:nilerr // malformed verifier output keeps findings unfiltered rather than failing the stage
|
||||
}
|
||||
unsupported := make(map[int]bool, len(captured.Verdicts))
|
||||
for _, v := range captured.Verdicts {
|
||||
if !v.Supported {
|
||||
unsupported[v.Index] = true
|
||||
}
|
||||
}
|
||||
|
||||
kept := make([]prreview.Finding, 0, len(findings))
|
||||
for i, f := range findings {
|
||||
if unsupported[i] {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, f)
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// adversaryVerdictsSchema is the adversary's verdict capture-tool schema.
|
||||
var adversaryVerdictsSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"verdicts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {"type": "integer"},
|
||||
"verdict": {"type": "string", "enum": ["confirmed", "challenged"]},
|
||||
"reason": {"type": "string"}
|
||||
},
|
||||
"required": ["index", "verdict"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["verdicts"]
|
||||
}`)
|
||||
|
||||
// adversarySkepticism is the extra instruction the adversary's mission
|
||||
// carries when intake scored the PR as likely machine-written, per the PRD.
|
||||
func adversarySkepticism(aiGenerated float64) string {
|
||||
if prreview.IsAIGenerated(aiGenerated) {
|
||||
return "This PR's own description reads as likely machine-written: be more sceptical " +
|
||||
"than usual, since a machine-written description is more likely to accompany " +
|
||||
"machine-written code with the same failure modes."
|
||||
}
|
||||
return "This PR's own description does not read as machine-written; review normally."
|
||||
}
|
||||
|
||||
// findingsListing renders every finding, numbered by its index, for a
|
||||
// mission that must reference findings back by that index.
|
||||
func findingsListing(findings []prreview.Finding) string {
|
||||
var listing strings.Builder
|
||||
for i, f := range findings {
|
||||
fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\nEvidence: %s\n\n",
|
||||
i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence)
|
||||
}
|
||||
return listing.String()
|
||||
}
|
||||
|
||||
// applyAdversaryVerdicts decodes one adversary_verdicts capture and sets each
|
||||
// referenced finding's Adversary field. An out-of-range index (malformed or
|
||||
// adversarial model output) is skipped rather than panicking.
|
||||
func applyAdversaryVerdicts(out []prreview.Finding, calls []json.RawMessage) {
|
||||
if len(calls) != 1 {
|
||||
return
|
||||
}
|
||||
var captured struct {
|
||||
Verdicts []struct {
|
||||
Index int `json:"index"`
|
||||
Verdict string `json:"verdict"`
|
||||
} `json:"verdicts"`
|
||||
}
|
||||
if json.Unmarshal(calls[0], &captured) != nil {
|
||||
return
|
||||
}
|
||||
for _, v := range captured.Verdicts {
|
||||
if v.Index < 0 || v.Index >= len(out) {
|
||||
continue
|
||||
}
|
||||
switch v.Verdict {
|
||||
case string(prreview.AdversaryConfirmed):
|
||||
out[v.Index].Adversary = prreview.AdversaryConfirmed
|
||||
case string(prreview.AdversaryChallenged):
|
||||
out[v.Index].Adversary = prreview.AdversaryChallenged
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runAdversary runs phase 5's adversary over every surviving finding at
|
||||
// once: it confirms or challenges each by index, and may report new findings
|
||||
// no reviewer caught. It reads intake's machine-written confidence and is
|
||||
// told to be more sceptical above the threshold, per the PRD. A failed or
|
||||
// malformed call leaves every finding's Adversary verdict unset (neither
|
||||
// confirmed nor challenged) and adds nothing -- an adversary that could not
|
||||
// run has reached no verdict, which is different from confirming everything.
|
||||
func runAdversary(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
|
||||
if len(findings) == 0 {
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
mission := fmt.Sprintf(
|
||||
"You are the adversary reviewing this pull request's findings. For each finding "+
|
||||
"below, confirm it (you agree it is a real defect) or challenge it (you believe "+
|
||||
"it is wrong or not actually a problem). Read the snapshot yourself to check. "+
|
||||
"%s Also look for any defect every reviewer missed and report it as a new "+
|
||||
"finding.\n\n%s\n"+
|
||||
"Call adversary_verdicts exactly once with one verdict per finding index. If you "+
|
||||
"found anything new, call report_findings once too (skip it if you found "+
|
||||
"nothing new). Then call finish with status \"passed\".",
|
||||
adversarySkepticism(tc.prReview.aiGenerated), findingsListing(findings),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "adversary", "review", mission, 25, []agentexec.CaptureTool{
|
||||
{
|
||||
Name: "adversary_verdicts", Description: "Record each finding's adversary verdict. Call exactly once, before finish.",
|
||||
Parameters: adversaryVerdictsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
|
||||
},
|
||||
reportFindingsTool,
|
||||
})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return findings, nil //nolint:nilerr // a call that could not run reached no verdict, which the pipeline treats as neutral, not as a stage failure
|
||||
}
|
||||
|
||||
out := append([]prreview.Finding{}, findings...)
|
||||
applyAdversaryVerdicts(out, res.Captures["adversary_verdicts"])
|
||||
if extra, err := decodeReportedFindings(res.Captures["report_findings"], "adversary"); err == nil {
|
||||
out = append(out, extra...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runCompoundCheck runs phase 5's compound-defect check once per cluster
|
||||
// that has more than one finding, in parallel, bounded the same as every
|
||||
// other pr-review fan-out. It flags the findings the agent names as
|
||||
// compounding (Finding.Compound = true), which is what scoring's compound
|
||||
// multiplier reads. A cluster with at most one finding cannot compound with
|
||||
// anything and is skipped without an agent call.
|
||||
func runCompoundCheck(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
|
||||
grouped := prreview.ClusterFindings(tc.prReview.clusters, findings)
|
||||
var clusterIDs []string
|
||||
for id, members := range grouped {
|
||||
if len(members) > 1 {
|
||||
clusterIDs = append(clusterIDs, id)
|
||||
}
|
||||
}
|
||||
if len(clusterIDs) == 0 {
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
flagged := make([]map[string]bool, len(clusterIDs))
|
||||
forEachBounded(prReviewConcurrency, len(clusterIDs), func(i int) {
|
||||
flagged[i] = runCompoundClusterCheck(ctx, tc, clusterIDs[i], grouped[clusterIDs[i]])
|
||||
})
|
||||
|
||||
compoundTitles := map[string]bool{}
|
||||
for _, set := range flagged {
|
||||
for title := range set {
|
||||
compoundTitles[title] = true
|
||||
}
|
||||
}
|
||||
out := append([]prreview.Finding{}, findings...)
|
||||
for i, f := range out {
|
||||
if compoundTitles[compoundKey(f)] {
|
||||
out[i].Compound = true
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// compoundKey is how runCompoundCheck matches a cluster agent's flagged
|
||||
// member back to a finding in the full list: findings carry no stable ID, so
|
||||
// (file, line, title) is the identity a compound call's answer is matched on.
|
||||
func compoundKey(f prreview.Finding) string {
|
||||
return fmt.Sprintf("%s:%d:%s", f.File, f.LineStart, f.Title)
|
||||
}
|
||||
|
||||
// runCompoundClusterCheck runs one phase-5 compound call over one cluster's
|
||||
// findings, returning the set of member keys (compoundKey) the agent flagged
|
||||
// as only defective in combination. A failed or malformed call flags nothing.
|
||||
func runCompoundClusterCheck(ctx context.Context, tc *TaskContext, clusterID string, members []prreview.Finding) map[string]bool {
|
||||
var listing strings.Builder
|
||||
for i, f := range members {
|
||||
fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\n\n", i, f.Severity, f.Title, f.File, f.LineStart, f.Body)
|
||||
}
|
||||
params := json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"indices": {"type": "array", "items": {"type": "integer"}}
|
||||
},
|
||||
"required": ["indices"]
|
||||
}`)
|
||||
mission := fmt.Sprintf(
|
||||
"These findings all touch the same cluster of files. Read them together: is "+
|
||||
"there a defect that only exists because of how two or more of them combine -- "+
|
||||
"one that neither finding describes on its own? If so, call compound_findings "+
|
||||
"once with the indices of the findings that compound (an empty array if none "+
|
||||
"do). Then call finish with status \"passed\".\n\n%s", listing.String(),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "compound-"+clusterID, "review", mission, 15, []agentexec.CaptureTool{{
|
||||
Name: "compound_findings", Description: "Record which findings compound. Call exactly once, before finish.",
|
||||
Parameters: params, RequiredFields: []string{"indices"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return nil //nolint:nilerr // a cluster call that could not run flags nothing rather than failing the whole compound-check stage
|
||||
}
|
||||
calls := res.Captures["compound_findings"]
|
||||
if len(calls) != 1 {
|
||||
return nil
|
||||
}
|
||||
var captured struct {
|
||||
Indices []int `json:"indices"`
|
||||
}
|
||||
if err := json.Unmarshal(calls[0], &captured); err != nil {
|
||||
return nil
|
||||
}
|
||||
flagged := make(map[string]bool, len(captured.Indices))
|
||||
for _, index := range captured.Indices {
|
||||
if index < 0 || index >= len(members) {
|
||||
continue
|
||||
}
|
||||
flagged[compoundKey(members[index])] = true
|
||||
}
|
||||
return flagged
|
||||
}
|
||||
|
||||
// prReviewGapRounds is phase 6's cap on gap-review rounds: at most 2, per the
|
||||
// PRD, so a coverage gap the gap reviewers cannot close does not loop forever.
|
||||
const prReviewGapRounds = 2
|
||||
|
||||
// stagePRCoverageConsistency is pipeline phase 6: the coverage gate and
|
||||
// consistency verification run in parallel and both append to
|
||||
// tc.prReview.findings, guarded by prReviewState.findingsMu
|
||||
// (docs/prds/pr-review-agent.md, phase 6: "Coverage and consistency, in
|
||||
// parallel").
|
||||
func stagePRCoverageConsistency() Stage {
|
||||
return Stage{Name: "coverage-consistency", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
var wg sync.WaitGroup
|
||||
var consistencyErr error
|
||||
wg.Go(func() {
|
||||
consistencyErr = runConsistencyVerification(ctx, tc)
|
||||
})
|
||||
runCoverageGate(ctx, tc)
|
||||
wg.Wait()
|
||||
return consistencyErr
|
||||
}}
|
||||
}
|
||||
|
||||
// runCoverageGate is phase 6's coverage gate: for up to prReviewGapRounds
|
||||
// rounds, find the clusters and high-exposure blast-radius files no finding
|
||||
// covers, run one gap reviewer per gap (reusing runReviewer, the same
|
||||
// reviewer definition phase 4 uses), and append whatever they find. It stops
|
||||
// early once nothing is uncovered, so a change that phase 4 already covered
|
||||
// fully never spends a gap-review call.
|
||||
func runCoverageGate(ctx context.Context, tc *TaskContext) {
|
||||
for range prReviewGapRounds {
|
||||
tc.prReview.findingsMu.Lock()
|
||||
uncoveredClusters, uncoveredFiles := prreview.CoverageGaps(tc.prReview.clusters, tc.prReview.highExposure, tc.prReview.findings)
|
||||
tc.prReview.findingsMu.Unlock()
|
||||
if len(uncoveredClusters) == 0 && len(uncoveredFiles) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
dims := gapDimensions(uncoveredClusters, uncoveredFiles)
|
||||
found := make([][]prreview.Finding, len(dims))
|
||||
forEachBounded(prReviewConcurrency, len(dims), func(i int) {
|
||||
found[i] = runReviewer(ctx, tc, dims[i])
|
||||
})
|
||||
|
||||
tc.prReview.findingsMu.Lock()
|
||||
for _, f := range found {
|
||||
tc.prReview.findings = append(tc.prReview.findings, f...)
|
||||
}
|
||||
tc.prReview.findingsMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// gapDimensions builds one reviewer dimension per coverage gap: a cluster's
|
||||
// files together (a cluster is reviewed as one unit, the way phase 4 would
|
||||
// have if a lens had proposed it), and one per uncovered high-exposure file.
|
||||
func gapDimensions(clusters []prreview.Cluster, files []string) []prreview.Dimension {
|
||||
dims := make([]prreview.Dimension, 0, len(clusters)+len(files))
|
||||
for _, cluster := range clusters {
|
||||
dims = append(dims, prreview.Dimension{
|
||||
Name: "gap-" + cluster.ID,
|
||||
Prompt: "This cluster of changed files was not reached by any review dimension. Review it for anything a reviewer would normally catch.",
|
||||
TargetFiles: cluster.Files,
|
||||
})
|
||||
}
|
||||
for _, file := range files {
|
||||
dims = append(dims, prreview.Dimension{
|
||||
Name: "gap-exposure-" + file,
|
||||
Prompt: "This file is reached by more than one changed file (high blast-radius exposure) but no reviewer covered it. Review it for anything the changes upstream may have broken here.",
|
||||
TargetFiles: []string{file},
|
||||
})
|
||||
}
|
||||
return dims
|
||||
}
|
||||
|
||||
// runConsistencyVerification is phase 6's consistency verification: one
|
||||
// agent call lists the cross-location obligations the changed code creates
|
||||
// and verifies each by reading both ends, reporting only the broken ones as
|
||||
// findings (the PRD's own wording -- "a broken obligation becomes a
|
||||
// finding"). It reuses report_findings, the same shape every other
|
||||
// findings-reporting call in the pipeline uses.
|
||||
func runConsistencyVerification(ctx context.Context, tc *TaskContext) error {
|
||||
mission := fmt.Sprintf(
|
||||
"Read this pull request's diff and the surrounding repository. List every "+
|
||||
"cross-location obligation the changed code creates: what must be true "+
|
||||
"elsewhere in the codebase for a changed line to be correct (a caller that "+
|
||||
"must pass a new argument, a schema a struct must still match, a doc that must "+
|
||||
"still describe the behaviour, an invariant a comment states that the new code "+
|
||||
"must uphold). For each obligation, verify it by reading both ends yourself. "+
|
||||
"Report only the obligations you found broken as findings -- an obligation you "+
|
||||
"checked and found intact is not a finding.\n\nDiff:\n%s\n\n"+
|
||||
"Call report_findings once (an empty array if every obligation you checked "+
|
||||
"held), then call finish with status \"passed\".",
|
||||
clip(tc.prReview.diff, 60000),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "consistency", "review", mission, 25, []agentexec.CaptureTool{reportFindingsTool})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return nil //nolint:nilerr // a consistency call that could not run reports no broken obligations rather than failing the stage
|
||||
}
|
||||
found, err := decodeReportedFindings(res.Captures["report_findings"], "consistency")
|
||||
if err != nil {
|
||||
return nil //nolint:nilerr // malformed consistency output reports no broken obligations rather than failing the stage
|
||||
}
|
||||
|
||||
tc.prReview.findingsMu.Lock()
|
||||
tc.prReview.findings = append(tc.prReview.findings, found...)
|
||||
tc.prReview.findingsMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
// writeSnapshotFile is a small helper for tests that need a real file on disk
|
||||
// for evidence extraction to read.
|
||||
func writeSnapshotFile(t *testing.T, dir, path, content string) {
|
||||
t.Helper()
|
||||
full := filepath.Join(dir, path)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRVerificationDropsUnsupportedHighPriorityFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n")
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
findings: []prreview.Finding{
|
||||
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityCritical, Title: "unsupported", Confidence: 0.9},
|
||||
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityNitpick, Title: "low priority", Confidence: 0.5},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
switch stage {
|
||||
case "evidence-verifier":
|
||||
return captureResult("verify_findings", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "supported": false}},
|
||||
}), nil
|
||||
case "adversary":
|
||||
return captureResult("adversary_verdicts", map[string]any{"verdicts": []map[string]any{}}), nil
|
||||
default:
|
||||
return passedResult("ok"), nil
|
||||
}
|
||||
}}
|
||||
|
||||
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("verification: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 {
|
||||
t.Fatalf("findings = %d, want 1 (the unsupported critical finding dropped)", len(tc.prReview.findings))
|
||||
}
|
||||
if tc.prReview.findings[0].Title != "low priority" {
|
||||
t.Errorf("surviving finding = %q, want the low-priority one the verifier never checked", tc.prReview.findings[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRVerificationAppliesAdversaryVerdicts(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n")
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
findings: []prreview.Finding{
|
||||
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityImportant, Title: "f0", Confidence: 0.9},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
switch stage {
|
||||
case "evidence-verifier":
|
||||
return captureResult("verify_findings", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "supported": true}},
|
||||
}), nil
|
||||
case "adversary":
|
||||
return captureResult("adversary_verdicts", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "verdict": "confirmed"}},
|
||||
}), nil
|
||||
default:
|
||||
return passedResult("ok"), nil
|
||||
}
|
||||
}}
|
||||
|
||||
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("verification: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 {
|
||||
t.Fatalf("findings = %d, want 1", len(tc.prReview.findings))
|
||||
}
|
||||
if tc.prReview.findings[0].Adversary != prreview.AdversaryConfirmed {
|
||||
t.Errorf("adversary verdict = %q, want confirmed", tc.prReview.findings[0].Adversary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRVerificationRunsNoAgentCallsWhenNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call expected with zero findings")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("verification: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRCompoundMarksFlaggedFindingsWithinCluster(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go", "b.go"}}},
|
||||
findings: []prreview.Finding{
|
||||
{File: "a.go", LineStart: 1, Title: "f0", Severity: prreview.SeverityImportant, Confidence: 0.9},
|
||||
{File: "b.go", LineStart: 1, Title: "f1", Severity: prreview.SeverityImportant, Confidence: 0.9},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
if stage == "compound-cluster_0" {
|
||||
return captureResult("compound_findings", map[string]any{"indices": []int{0, 1}}), nil
|
||||
}
|
||||
return passedResult("ok"), nil
|
||||
}}
|
||||
|
||||
findings, err := runCompoundCheck(context.Background(), tc, tc.prReview.findings)
|
||||
if err != nil {
|
||||
t.Fatalf("runCompoundCheck: %v", err)
|
||||
}
|
||||
for _, f := range findings {
|
||||
if !f.Compound {
|
||||
t.Errorf("finding %q Compound = false, want true (both findings in the flagged cluster)", f.Title)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRCoverageConsistencyRunsGapReviewersAndConsistencyInParallel(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
dir := t.TempDir()
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: dir,
|
||||
clusters: []prreview.Cluster{{ID: "cluster_0", Name: "pkg", Files: []string{"pkg/a.go"}}},
|
||||
highExposure: []string{"pkg/shared.go"},
|
||||
findings: nil,
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
switch stage {
|
||||
case "consistency":
|
||||
return captureResult("report_findings", map[string]any{
|
||||
"findings": []map[string]any{
|
||||
{"file": "pkg/a.go", "line_start": 1, "severity": "important", "title": "broken obligation", "body": "b", "evidence": "e", "confidence": 0.8},
|
||||
},
|
||||
}), nil
|
||||
default:
|
||||
// Gap reviewers reuse runReviewer's report_findings shape.
|
||||
return captureResult("report_findings", map[string]any{"findings": []any{}}), nil
|
||||
}
|
||||
}}
|
||||
|
||||
if err := stagePRCoverageConsistency().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("coverage-consistency: %v", err)
|
||||
}
|
||||
|
||||
foundConsistency := false
|
||||
for _, f := range tc.prReview.findings {
|
||||
if f.Title == "broken obligation" {
|
||||
foundConsistency = true
|
||||
}
|
||||
}
|
||||
if !foundConsistency {
|
||||
t.Error("consistency verification's finding was not appended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoverageGapCapsAtTwoRounds(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go"}}},
|
||||
}
|
||||
var calls int
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
calls++
|
||||
// Every gap reviewer call reports nothing, so the cluster stays
|
||||
// uncovered and a naive loop would run forever without the cap.
|
||||
return captureResult("report_findings", map[string]any{"findings": []any{}}), nil
|
||||
}}
|
||||
|
||||
runCoverageGate(context.Background(), tc)
|
||||
|
||||
if calls != 2 {
|
||||
t.Errorf("gap reviewer calls = %d, want exactly 2 (the round cap)", calls)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user