feat(workflow): wire pr-review's verification and coverage/consistency stages

Adds phases 5 and 6 of docs/prds/pr-review-agent.md between the existing
review and synthesis stages: an evidence verifier that drops unsupported
high-priority findings, an adversary that confirms/challenges findings and
may report new ones (more sceptical for a likely machine-written PR), a
per-cluster compound-defect check, a coverage gate that runs gap reviewers
for uncovered clusters and high-exposure blast-radius files (capped at 2
rounds), and consistency verification for cross-location obligations. The
coverage gate and consistency verification run in parallel per the PRD,
guarded by prReviewState.findingsMu.

Phases 5/6 are afbk.4's scope; the merge gate (phase 8) remains afbk.5's.
This commit is contained in:
sam committed 2026-09-28 11:04:44 +10:00
1 parent b4a174c629
commit 7324617cd1
3 files changed
+765 -41

No files matched your search

+82 -41
View File
@@ -58,11 +58,23 @@ type prReviewState struct {
snapshotDir string
headSHA string
blastRadius []string
// clusters and highExposure are phase 6's coverage-gate inputs, computed
// once in anatomy alongside blastRadius: clusters groups changed files by
// directory, highExposure is the blastRadius files more than one changed
// file reaches.
clusters []prreview.Cluster
highExposure []string
narrative string
dimensions []prreview.Dimension
findings []prreview.Finding
scored []prreview.ScoredFinding
// findingsMu guards findings during phase 6, where the coverage gate and
// consistency verification append to it from two goroutines running in
// parallel (docs/prds/pr-review-agent.md, phase 6). Every other phase
// touches findings from a single goroutine and does not need it.
findingsMu sync.Mutex
}
// prReviewConcurrency bounds phases 3 (lenses), 4 (reviewers) and 9 (polish),
@@ -71,9 +83,9 @@ type prReviewState struct {
const prReviewConcurrency = 8
// PRReview is the pull request reviewer: a fixed pipeline, not a single
// agent (docs/prds/pr-review-agent.md). This bead wires phases 1, 2, 3, 4, 7
// and 9; phases 5, 6 and 8 (verification, coverage/consistency, merge gate)
// are archie-core-afbk.4 and .5.
// agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2,
// 3, 4, 7 and 9; archie-core-afbk.4 adds 5 and 6 (verification,
// coverage/consistency); phase 8 (the merge gate) is archie-core-afbk.5.
func PRReview() Workflow {
return Workflow{
Name: "pr-review",
@@ -82,6 +94,8 @@ func PRReview() Workflow {
stagePRAnatomy(),
stagePRLenses(),
stagePRReview(),
stagePRVerification(),
stagePRCoverageConsistency(),
stagePRSynthesis(),
stagePROutput(),
},
@@ -197,6 +211,12 @@ func stagePRAnatomy() Stage {
return fmt.Errorf("compute blast radius: %w", err)
}
tc.prReview.blastRadius = blast
tc.prReview.clusters = prreview.ClusterFiles(tc.prReview.files)
exposure, err := prreview.ExposureCounts(os.DirFS(dir), changed)
if err != nil {
return fmt.Errorf("compute exposure counts: %w", err)
}
tc.prReview.highExposure = prreview.HighExposureFiles(exposure)
mission := fmt.Sprintf(
"Read this pull request's diff and enough of the surrounding repository to "+
@@ -349,31 +369,6 @@ func stagePRReview() Stage {
// case a distinct StepFailed child step, so a reader of the execution tree
// (not this slice) can tell "unreviewed" from "reviewed-clean" apart.
func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) []prreview.Finding {
params := json.RawMessage(`{
"type": "object",
"properties": {
"findings": {
"type": "array",
"items": {
"type": "object",
"properties": {
"file": {"type": "string"},
"line_start": {"type": "integer"},
"line_end": {"type": "integer"},
"severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]},
"title": {"type": "string"},
"body": {"type": "string"},
"suggestion": {"type": "string"},
"evidence": {"type": "string"},
"confidence": {"type": "number"},
"tags": {"type": "array", "items": {"type": "string"}}
},
"required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"]
}
}
},
"required": ["findings"]
}`)
mission := fmt.Sprintf(
"%s\n\nTarget files: %s\n\nRead the target files (and, if useful, the context "+
"files) in the snapshot and report every finding for this dimension. Quote the "+
@@ -383,17 +378,63 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [
dim.Prompt, strings.Join(dim.TargetFiles, ", "),
)
name := "reviewer-" + dim.Name
res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{{
Name: "report_findings", Description: "Record this dimension's findings. Call exactly once, before finish.",
Parameters: params, RequiredFields: []string{"findings"}, MaxCalls: 1,
}})
res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{reportFindingsTool})
if runErr != nil || res.Status != agentexec.StatusPassed {
return nil
}
calls := res.Captures["report_findings"]
if len(calls) != 1 {
findings, err := decodeReportedFindings(res.Captures["report_findings"], dim.Name)
if err != nil {
return nil
}
return findings
}
// reportFindingsSchema is the finding shape every pr-review agent call that
// reports findings captures them in: phase 4's reviewer, phase 6's gap
// reviewers (which reuse runReviewer directly) and consistency verification.
var reportFindingsSchema = json.RawMessage(`{
"type": "object",
"properties": {
"findings": {
"type": "array",
"items": {
"type": "object",
"properties": {
"file": {"type": "string"},
"line_start": {"type": "integer"},
"line_end": {"type": "integer"},
"severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]},
"title": {"type": "string"},
"body": {"type": "string"},
"suggestion": {"type": "string"},
"evidence": {"type": "string"},
"confidence": {"type": "number"},
"tags": {"type": "array", "items": {"type": "string"}}
},
"required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"]
}
}
},
"required": ["findings"]
}`)
// reportFindingsTool is the capture tool every findings-reporting call
// registers, at the shape reportFindingsSchema names.
var reportFindingsTool = agentexec.CaptureTool{
Name: "report_findings", Description: "Record findings. Call exactly once, before finish.",
Parameters: reportFindingsSchema, RequiredFields: []string{"findings"}, MaxCalls: 1,
}
// decodeReportedFindings decodes one report_findings capture into findings
// tagged with dimension, shared by every call site reportFindingsSchema
// backs. A call that never reported (no captures) or reported malformed JSON
// decodes to no findings and an error the caller treats as "nothing to add",
// not as a stage failure -- an agent that could not report is not evidence
// the code that follows should stop.
func decodeReportedFindings(calls []json.RawMessage, dimension string) ([]prreview.Finding, error) {
if len(calls) != 1 {
return nil, fmt.Errorf("report_findings called %d times (want exactly once)", len(calls))
}
var captured struct {
Findings []struct {
File string `json:"file"`
@@ -409,7 +450,7 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [
} `json:"findings"`
}
if err := json.Unmarshal(calls[0], &captured); err != nil {
return nil
return nil, fmt.Errorf("decode findings: %w", err)
}
out := make([]prreview.Finding, len(captured.Findings))
for i, f := range captured.Findings {
@@ -418,12 +459,12 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [
lineEnd = f.LineStart
}
out[i] = prreview.Finding{
Dimension: dim.Name, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd,
Dimension: dimension, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd,
Severity: prreview.Severity(f.Severity), Title: f.Title, Body: f.Body,
Suggestion: f.Suggestion, Evidence: f.Evidence, Confidence: f.Confidence, Tags: f.Tags,
}
}
return out
return out, nil
}
// prReviewMaxSteps bounds one reviewer's tool-loop iterations. Exhausting it
@@ -475,10 +516,10 @@ func runReviewerAgent(
// stagePRSynthesis is pipeline phase 7, code only: score, drop findings
// under their severity's confidence floor, merge duplicates, rank, and cap
// at the inline comment limit. Phases 5, 6 and 8 (verification, coverage and
// consistency, the merge gate) do not run yet, so every surviving finding is
// scored on phase 4's raw output alone -- no compound, adversary or
// merge-gate multiplier has anything to apply yet.
// at the inline comment limit. Phase 8 (the merge gate) does not run yet, so
// every surviving finding's Blocking flag is whatever phase 5/6 left it as
// (false, since neither sets it -- that is archie-core-afbk.5's decision to
// make).
func stagePRSynthesis() Stage {
return Stage{Name: "synthesis", Run: func(_ context.Context, tc *TaskContext) error {
scored := prreview.Score(tc.prReview.findings, prreview.ScoreInputs{
@@ -0,0 +1,488 @@
package workflow
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"sync"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
)
// stagePRVerification is pipeline phase 5, over the review findings phase 4
// produced: extract each finding's evidence package (code), check the
// high-priority findings against theirs (an evidence verifier agent), have
// an adversary confirm or challenge every finding and add anything every
// reviewer missed, then flag within-cluster compound defects
// (docs/prds/pr-review-agent.md, phase 5).
func stagePRVerification() Stage {
return Stage{Name: "verification", Run: func(ctx context.Context, tc *TaskContext) error {
if len(tc.prReview.findings) == 0 {
return nil
}
verified, err := runEvidenceVerifier(ctx, tc, tc.prReview.findings)
if err != nil {
return fmt.Errorf("evidence verifier: %w", err)
}
confirmed, err := runAdversary(ctx, tc, verified)
if err != nil {
return fmt.Errorf("adversary: %w", err)
}
compounded, err := runCompoundCheck(ctx, tc, confirmed)
if err != nil {
return fmt.Errorf("compound check: %w", err)
}
tc.prReview.findings = compounded
return nil
}}
}
// evidencePackageFor extracts a finding's evidence package from the
// snapshot, reusing afbk.1's ExtractEvidence. A file the snapshot no longer
// has (a finding pointing at a path that does not exist) extracts an empty
// package rather than failing the stage: that emptiness is itself the
// evidence verifier's answer.
func evidencePackageFor(tc *TaskContext, f prreview.Finding) prreview.EvidencePackage {
pkg, err := prreview.ExtractEvidence(os.DirFS(tc.prReview.snapshotDir), prreview.EvidenceRequest{
File: f.File, LineStart: f.LineStart, LineEnd: f.LineEnd,
Title: f.Title, Body: f.Body, Evidence: f.Evidence,
})
if err != nil {
return prreview.EvidencePackage{FindingTitle: f.Title, File: f.File}
}
return pkg
}
// verifyFindingsSchema is the evidence verifier's capture-tool schema.
var verifyFindingsSchema = json.RawMessage(`{
"type": "object",
"properties": {
"verdicts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"index": {"type": "integer"},
"supported": {"type": "boolean"},
"reason": {"type": "string"}
},
"required": ["index", "supported"]
}
}
},
"required": ["verdicts"]
}`)
// evidenceCandidate is one high-priority finding the evidence verifier is
// asked to check, paired with its index in the caller's original slice: the
// verifier's answer references findings by that index, not by content.
type evidenceCandidate struct {
index int
finding prreview.Finding
}
// highPriorityCandidates picks the findings runEvidenceVerifier checks:
// every high-priority finding, each still carrying its position in findings.
func highPriorityCandidates(findings []prreview.Finding) []evidenceCandidate {
var candidates []evidenceCandidate
for i, f := range findings {
if prreview.IsHighPriority(f) {
candidates = append(candidates, evidenceCandidate{index: i, finding: f})
}
}
return candidates
}
// evidenceListing renders each candidate's claim and evidence package as the
// evidence verifier's mission text.
func evidenceListing(tc *TaskContext, candidates []evidenceCandidate) string {
var listing strings.Builder
for _, c := range candidates {
pkg := evidencePackageFor(tc, c.finding)
fmt.Fprintf(&listing, "Finding %d: %s (%s:%d)\nClaim: %s\nQuoted evidence: %s\nCode at the cited lines:\n%s\n\n",
c.index, c.finding.Title, c.finding.File, c.finding.LineStart, c.finding.Body, c.finding.Evidence, pkg.PrimaryCode)
}
return listing.String()
}
// runEvidenceVerifier runs phase 5's evidence verifier over every
// high-priority (critical or important) finding, dropping the ones it finds
// unsupported by their own evidence package. A finding below high priority is
// never sent, and always kept: the verifier exists to catch a hallucinated
// critical, not to grade a nitpick. A failed or malformed verifier call keeps
// every finding unfiltered -- the pipeline is recall-first, so a verifier
// that could not run is not evidence any finding is wrong; the error is
// intentionally swallowed here, not propagated.
func runEvidenceVerifier(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
candidates := highPriorityCandidates(findings)
if len(candidates) == 0 {
return findings, nil
}
mission := fmt.Sprintf(
"Each finding below cites code as evidence for its claim. Check whether the "+
"quoted code at the cited lines actually supports the claim -- not whether "+
"the claim is a good catch, only whether the evidence is real and says what "+
"the finding says it says.\n\n%s\n"+
"Call verify_findings exactly once with one verdict per finding index, then "+
"call finish with status \"passed\".",
evidenceListing(tc, candidates),
)
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "evidence-verifier", "review", mission, 15, []agentexec.CaptureTool{{
Name: "verify_findings", Description: "Record each finding's evidence verdict. Call exactly once, before finish.",
Parameters: verifyFindingsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
}})
if err != nil || res.Status != agentexec.StatusPassed {
return findings, nil //nolint:nilerr // recall-first: a verifier that could not run is not evidence any finding is wrong, so nothing is filtered
}
calls := res.Captures["verify_findings"]
if len(calls) != 1 {
return findings, nil
}
var captured struct {
Verdicts []struct {
Index int `json:"index"`
Supported bool `json:"supported"`
} `json:"verdicts"`
}
if err := json.Unmarshal(calls[0], &captured); err != nil {
return findings, nil //nolint:nilerr // malformed verifier output keeps findings unfiltered rather than failing the stage
}
unsupported := make(map[int]bool, len(captured.Verdicts))
for _, v := range captured.Verdicts {
if !v.Supported {
unsupported[v.Index] = true
}
}
kept := make([]prreview.Finding, 0, len(findings))
for i, f := range findings {
if unsupported[i] {
continue
}
kept = append(kept, f)
}
return kept, nil
}
// adversaryVerdictsSchema is the adversary's verdict capture-tool schema.
var adversaryVerdictsSchema = json.RawMessage(`{
"type": "object",
"properties": {
"verdicts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"index": {"type": "integer"},
"verdict": {"type": "string", "enum": ["confirmed", "challenged"]},
"reason": {"type": "string"}
},
"required": ["index", "verdict"]
}
}
},
"required": ["verdicts"]
}`)
// adversarySkepticism is the extra instruction the adversary's mission
// carries when intake scored the PR as likely machine-written, per the PRD.
func adversarySkepticism(aiGenerated float64) string {
if prreview.IsAIGenerated(aiGenerated) {
return "This PR's own description reads as likely machine-written: be more sceptical " +
"than usual, since a machine-written description is more likely to accompany " +
"machine-written code with the same failure modes."
}
return "This PR's own description does not read as machine-written; review normally."
}
// findingsListing renders every finding, numbered by its index, for a
// mission that must reference findings back by that index.
func findingsListing(findings []prreview.Finding) string {
var listing strings.Builder
for i, f := range findings {
fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\nEvidence: %s\n\n",
i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence)
}
return listing.String()
}
// applyAdversaryVerdicts decodes one adversary_verdicts capture and sets each
// referenced finding's Adversary field. An out-of-range index (malformed or
// adversarial model output) is skipped rather than panicking.
func applyAdversaryVerdicts(out []prreview.Finding, calls []json.RawMessage) {
if len(calls) != 1 {
return
}
var captured struct {
Verdicts []struct {
Index int `json:"index"`
Verdict string `json:"verdict"`
} `json:"verdicts"`
}
if json.Unmarshal(calls[0], &captured) != nil {
return
}
for _, v := range captured.Verdicts {
if v.Index < 0 || v.Index >= len(out) {
continue
}
switch v.Verdict {
case string(prreview.AdversaryConfirmed):
out[v.Index].Adversary = prreview.AdversaryConfirmed
case string(prreview.AdversaryChallenged):
out[v.Index].Adversary = prreview.AdversaryChallenged
}
}
}
// runAdversary runs phase 5's adversary over every surviving finding at
// once: it confirms or challenges each by index, and may report new findings
// no reviewer caught. It reads intake's machine-written confidence and is
// told to be more sceptical above the threshold, per the PRD. A failed or
// malformed call leaves every finding's Adversary verdict unset (neither
// confirmed nor challenged) and adds nothing -- an adversary that could not
// run has reached no verdict, which is different from confirming everything.
func runAdversary(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
if len(findings) == 0 {
return findings, nil
}
mission := fmt.Sprintf(
"You are the adversary reviewing this pull request's findings. For each finding "+
"below, confirm it (you agree it is a real defect) or challenge it (you believe "+
"it is wrong or not actually a problem). Read the snapshot yourself to check. "+
"%s Also look for any defect every reviewer missed and report it as a new "+
"finding.\n\n%s\n"+
"Call adversary_verdicts exactly once with one verdict per finding index. If you "+
"found anything new, call report_findings once too (skip it if you found "+
"nothing new). Then call finish with status \"passed\".",
adversarySkepticism(tc.prReview.aiGenerated), findingsListing(findings),
)
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "adversary", "review", mission, 25, []agentexec.CaptureTool{
{
Name: "adversary_verdicts", Description: "Record each finding's adversary verdict. Call exactly once, before finish.",
Parameters: adversaryVerdictsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
},
reportFindingsTool,
})
if err != nil || res.Status != agentexec.StatusPassed {
return findings, nil //nolint:nilerr // a call that could not run reached no verdict, which the pipeline treats as neutral, not as a stage failure
}
out := append([]prreview.Finding{}, findings...)
applyAdversaryVerdicts(out, res.Captures["adversary_verdicts"])
if extra, err := decodeReportedFindings(res.Captures["report_findings"], "adversary"); err == nil {
out = append(out, extra...)
}
return out, nil
}
// runCompoundCheck runs phase 5's compound-defect check once per cluster
// that has more than one finding, in parallel, bounded the same as every
// other pr-review fan-out. It flags the findings the agent names as
// compounding (Finding.Compound = true), which is what scoring's compound
// multiplier reads. A cluster with at most one finding cannot compound with
// anything and is skipped without an agent call.
func runCompoundCheck(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) {
grouped := prreview.ClusterFindings(tc.prReview.clusters, findings)
var clusterIDs []string
for id, members := range grouped {
if len(members) > 1 {
clusterIDs = append(clusterIDs, id)
}
}
if len(clusterIDs) == 0 {
return findings, nil
}
flagged := make([]map[string]bool, len(clusterIDs))
forEachBounded(prReviewConcurrency, len(clusterIDs), func(i int) {
flagged[i] = runCompoundClusterCheck(ctx, tc, clusterIDs[i], grouped[clusterIDs[i]])
})
compoundTitles := map[string]bool{}
for _, set := range flagged {
for title := range set {
compoundTitles[title] = true
}
}
out := append([]prreview.Finding{}, findings...)
for i, f := range out {
if compoundTitles[compoundKey(f)] {
out[i].Compound = true
}
}
return out, nil
}
// compoundKey is how runCompoundCheck matches a cluster agent's flagged
// member back to a finding in the full list: findings carry no stable ID, so
// (file, line, title) is the identity a compound call's answer is matched on.
func compoundKey(f prreview.Finding) string {
return fmt.Sprintf("%s:%d:%s", f.File, f.LineStart, f.Title)
}
// runCompoundClusterCheck runs one phase-5 compound call over one cluster's
// findings, returning the set of member keys (compoundKey) the agent flagged
// as only defective in combination. A failed or malformed call flags nothing.
func runCompoundClusterCheck(ctx context.Context, tc *TaskContext, clusterID string, members []prreview.Finding) map[string]bool {
var listing strings.Builder
for i, f := range members {
fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\n\n", i, f.Severity, f.Title, f.File, f.LineStart, f.Body)
}
params := json.RawMessage(`{
"type": "object",
"properties": {
"indices": {"type": "array", "items": {"type": "integer"}}
},
"required": ["indices"]
}`)
mission := fmt.Sprintf(
"These findings all touch the same cluster of files. Read them together: is "+
"there a defect that only exists because of how two or more of them combine -- "+
"one that neither finding describes on its own? If so, call compound_findings "+
"once with the indices of the findings that compound (an empty array if none "+
"do). Then call finish with status \"passed\".\n\n%s", listing.String(),
)
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "compound-"+clusterID, "review", mission, 15, []agentexec.CaptureTool{{
Name: "compound_findings", Description: "Record which findings compound. Call exactly once, before finish.",
Parameters: params, RequiredFields: []string{"indices"}, MaxCalls: 1,
}})
if err != nil || res.Status != agentexec.StatusPassed {
return nil //nolint:nilerr // a cluster call that could not run flags nothing rather than failing the whole compound-check stage
}
calls := res.Captures["compound_findings"]
if len(calls) != 1 {
return nil
}
var captured struct {
Indices []int `json:"indices"`
}
if err := json.Unmarshal(calls[0], &captured); err != nil {
return nil
}
flagged := make(map[string]bool, len(captured.Indices))
for _, index := range captured.Indices {
if index < 0 || index >= len(members) {
continue
}
flagged[compoundKey(members[index])] = true
}
return flagged
}
// prReviewGapRounds is phase 6's cap on gap-review rounds: at most 2, per the
// PRD, so a coverage gap the gap reviewers cannot close does not loop forever.
const prReviewGapRounds = 2
// stagePRCoverageConsistency is pipeline phase 6: the coverage gate and
// consistency verification run in parallel and both append to
// tc.prReview.findings, guarded by prReviewState.findingsMu
// (docs/prds/pr-review-agent.md, phase 6: "Coverage and consistency, in
// parallel").
func stagePRCoverageConsistency() Stage {
return Stage{Name: "coverage-consistency", Run: func(ctx context.Context, tc *TaskContext) error {
var wg sync.WaitGroup
var consistencyErr error
wg.Go(func() {
consistencyErr = runConsistencyVerification(ctx, tc)
})
runCoverageGate(ctx, tc)
wg.Wait()
return consistencyErr
}}
}
// runCoverageGate is phase 6's coverage gate: for up to prReviewGapRounds
// rounds, find the clusters and high-exposure blast-radius files no finding
// covers, run one gap reviewer per gap (reusing runReviewer, the same
// reviewer definition phase 4 uses), and append whatever they find. It stops
// early once nothing is uncovered, so a change that phase 4 already covered
// fully never spends a gap-review call.
func runCoverageGate(ctx context.Context, tc *TaskContext) {
for range prReviewGapRounds {
tc.prReview.findingsMu.Lock()
uncoveredClusters, uncoveredFiles := prreview.CoverageGaps(tc.prReview.clusters, tc.prReview.highExposure, tc.prReview.findings)
tc.prReview.findingsMu.Unlock()
if len(uncoveredClusters) == 0 && len(uncoveredFiles) == 0 {
return
}
dims := gapDimensions(uncoveredClusters, uncoveredFiles)
found := make([][]prreview.Finding, len(dims))
forEachBounded(prReviewConcurrency, len(dims), func(i int) {
found[i] = runReviewer(ctx, tc, dims[i])
})
tc.prReview.findingsMu.Lock()
for _, f := range found {
tc.prReview.findings = append(tc.prReview.findings, f...)
}
tc.prReview.findingsMu.Unlock()
}
}
// gapDimensions builds one reviewer dimension per coverage gap: a cluster's
// files together (a cluster is reviewed as one unit, the way phase 4 would
// have if a lens had proposed it), and one per uncovered high-exposure file.
func gapDimensions(clusters []prreview.Cluster, files []string) []prreview.Dimension {
dims := make([]prreview.Dimension, 0, len(clusters)+len(files))
for _, cluster := range clusters {
dims = append(dims, prreview.Dimension{
Name: "gap-" + cluster.ID,
Prompt: "This cluster of changed files was not reached by any review dimension. Review it for anything a reviewer would normally catch.",
TargetFiles: cluster.Files,
})
}
for _, file := range files {
dims = append(dims, prreview.Dimension{
Name: "gap-exposure-" + file,
Prompt: "This file is reached by more than one changed file (high blast-radius exposure) but no reviewer covered it. Review it for anything the changes upstream may have broken here.",
TargetFiles: []string{file},
})
}
return dims
}
// runConsistencyVerification is phase 6's consistency verification: one
// agent call lists the cross-location obligations the changed code creates
// and verifies each by reading both ends, reporting only the broken ones as
// findings (the PRD's own wording -- "a broken obligation becomes a
// finding"). It reuses report_findings, the same shape every other
// findings-reporting call in the pipeline uses.
func runConsistencyVerification(ctx context.Context, tc *TaskContext) error {
mission := fmt.Sprintf(
"Read this pull request's diff and the surrounding repository. List every "+
"cross-location obligation the changed code creates: what must be true "+
"elsewhere in the codebase for a changed line to be correct (a caller that "+
"must pass a new argument, a schema a struct must still match, a doc that must "+
"still describe the behaviour, an invariant a comment states that the new code "+
"must uphold). For each obligation, verify it by reading both ends yourself. "+
"Report only the obligations you found broken as findings -- an obligation you "+
"checked and found intact is not a finding.\n\nDiff:\n%s\n\n"+
"Call report_findings once (an empty array if every obligation you checked "+
"held), then call finish with status \"passed\".",
clip(tc.prReview.diff, 60000),
)
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "consistency", "review", mission, 25, []agentexec.CaptureTool{reportFindingsTool})
if err != nil || res.Status != agentexec.StatusPassed {
return nil //nolint:nilerr // a consistency call that could not run reports no broken obligations rather than failing the stage
}
found, err := decodeReportedFindings(res.Captures["report_findings"], "consistency")
if err != nil {
return nil //nolint:nilerr // malformed consistency output reports no broken obligations rather than failing the stage
}
tc.prReview.findingsMu.Lock()
tc.prReview.findings = append(tc.prReview.findings, found...)
tc.prReview.findingsMu.Unlock()
return nil
}
@@ -0,0 +1,195 @@
package workflow
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
)
// writeSnapshotFile is a small helper for tests that need a real file on disk
// for evidence extraction to read.
func writeSnapshotFile(t *testing.T, dir, path, content string) {
t.Helper()
full := filepath.Join(dir, path)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
func TestStagePRVerificationDropsUnsupportedHighPriorityFindings(t *testing.T) {
tc := baseTaskContext(t)
dir := t.TempDir()
writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n")
tc.prReview = &prReviewState{
snapshotDir: dir,
findings: []prreview.Finding{
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityCritical, Title: "unsupported", Confidence: 0.9},
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityNitpick, Title: "low priority", Confidence: 0.5},
},
}
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
switch stage {
case "evidence-verifier":
return captureResult("verify_findings", map[string]any{
"verdicts": []map[string]any{{"index": 0, "supported": false}},
}), nil
case "adversary":
return captureResult("adversary_verdicts", map[string]any{"verdicts": []map[string]any{}}), nil
default:
return passedResult("ok"), nil
}
}}
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
t.Fatalf("verification: %v", err)
}
if len(tc.prReview.findings) != 1 {
t.Fatalf("findings = %d, want 1 (the unsupported critical finding dropped)", len(tc.prReview.findings))
}
if tc.prReview.findings[0].Title != "low priority" {
t.Errorf("surviving finding = %q, want the low-priority one the verifier never checked", tc.prReview.findings[0].Title)
}
}
func TestStagePRVerificationAppliesAdversaryVerdicts(t *testing.T) {
tc := baseTaskContext(t)
dir := t.TempDir()
writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n")
tc.prReview = &prReviewState{
snapshotDir: dir,
findings: []prreview.Finding{
{File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityImportant, Title: "f0", Confidence: 0.9},
},
}
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
switch stage {
case "evidence-verifier":
return captureResult("verify_findings", map[string]any{
"verdicts": []map[string]any{{"index": 0, "supported": true}},
}), nil
case "adversary":
return captureResult("adversary_verdicts", map[string]any{
"verdicts": []map[string]any{{"index": 0, "verdict": "confirmed"}},
}), nil
default:
return passedResult("ok"), nil
}
}}
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
t.Fatalf("verification: %v", err)
}
if len(tc.prReview.findings) != 1 {
t.Fatalf("findings = %d, want 1", len(tc.prReview.findings))
}
if tc.prReview.findings[0].Adversary != prreview.AdversaryConfirmed {
t.Errorf("adversary verdict = %q, want confirmed", tc.prReview.findings[0].Adversary)
}
}
func TestStagePRVerificationRunsNoAgentCallsWhenNoFindings(t *testing.T) {
tc := baseTaskContext(t)
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
t.Fatal("no agent call expected with zero findings")
return agentexec.Result{}, nil
}}
if err := stagePRVerification().Run(context.Background(), tc); err != nil {
t.Fatalf("verification: %v", err)
}
}
func TestStagePRCompoundMarksFlaggedFindingsWithinCluster(t *testing.T) {
tc := baseTaskContext(t)
dir := t.TempDir()
tc.prReview = &prReviewState{
snapshotDir: dir,
clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go", "b.go"}}},
findings: []prreview.Finding{
{File: "a.go", LineStart: 1, Title: "f0", Severity: prreview.SeverityImportant, Confidence: 0.9},
{File: "b.go", LineStart: 1, Title: "f1", Severity: prreview.SeverityImportant, Confidence: 0.9},
},
}
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
if stage == "compound-cluster_0" {
return captureResult("compound_findings", map[string]any{"indices": []int{0, 1}}), nil
}
return passedResult("ok"), nil
}}
findings, err := runCompoundCheck(context.Background(), tc, tc.prReview.findings)
if err != nil {
t.Fatalf("runCompoundCheck: %v", err)
}
for _, f := range findings {
if !f.Compound {
t.Errorf("finding %q Compound = false, want true (both findings in the flagged cluster)", f.Title)
}
}
}
func TestStagePRCoverageConsistencyRunsGapReviewersAndConsistencyInParallel(t *testing.T) {
tc := baseTaskContext(t)
dir := t.TempDir()
tc.prReview = &prReviewState{
snapshotDir: dir,
clusters: []prreview.Cluster{{ID: "cluster_0", Name: "pkg", Files: []string{"pkg/a.go"}}},
highExposure: []string{"pkg/shared.go"},
findings: nil,
}
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
switch stage {
case "consistency":
return captureResult("report_findings", map[string]any{
"findings": []map[string]any{
{"file": "pkg/a.go", "line_start": 1, "severity": "important", "title": "broken obligation", "body": "b", "evidence": "e", "confidence": 0.8},
},
}), nil
default:
// Gap reviewers reuse runReviewer's report_findings shape.
return captureResult("report_findings", map[string]any{"findings": []any{}}), nil
}
}}
if err := stagePRCoverageConsistency().Run(context.Background(), tc); err != nil {
t.Fatalf("coverage-consistency: %v", err)
}
foundConsistency := false
for _, f := range tc.prReview.findings {
if f.Title == "broken obligation" {
foundConsistency = true
}
}
if !foundConsistency {
t.Error("consistency verification's finding was not appended")
}
}
func TestCoverageGapCapsAtTwoRounds(t *testing.T) {
tc := baseTaskContext(t)
tc.prReview = &prReviewState{
snapshotDir: t.TempDir(),
clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go"}}},
}
var calls int
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
calls++
// Every gap reviewer call reports nothing, so the cluster stays
// uncovered and a naive loop would run forever without the cap.
return captureResult("report_findings", map[string]any{"findings": []any{}}), nil
}}
runCoverageGate(context.Background(), tc)
if calls != 2 {
t.Errorf("gap reviewer calls = %d, want exactly 2 (the round cap)", calls)
}
}