diff --git a/internal/domain/workflow/prreview_stages.go b/internal/domain/workflow/prreview_stages.go index 92089bbfd..3be10b93e 100644 --- a/internal/domain/workflow/prreview_stages.go +++ b/internal/domain/workflow/prreview_stages.go @@ -58,11 +58,23 @@ type prReviewState struct { snapshotDir string headSHA string blastRadius []string + // clusters and highExposure are phase 6's coverage-gate inputs, computed + // once in anatomy alongside blastRadius: clusters groups changed files by + // directory, highExposure is the blastRadius files more than one changed + // file reaches. + clusters []prreview.Cluster + highExposure []string narrative string dimensions []prreview.Dimension findings []prreview.Finding scored []prreview.ScoredFinding + + // findingsMu guards findings during phase 6, where the coverage gate and + // consistency verification append to it from two goroutines running in + // parallel (docs/prds/pr-review-agent.md, phase 6). Every other phase + // touches findings from a single goroutine and does not need it. + findingsMu sync.Mutex } // prReviewConcurrency bounds phases 3 (lenses), 4 (reviewers) and 9 (polish), @@ -71,9 +83,9 @@ type prReviewState struct { const prReviewConcurrency = 8 // PRReview is the pull request reviewer: a fixed pipeline, not a single -// agent (docs/prds/pr-review-agent.md). This bead wires phases 1, 2, 3, 4, 7 -// and 9; phases 5, 6 and 8 (verification, coverage/consistency, merge gate) -// are archie-core-afbk.4 and .5. +// agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2, +// 3, 4, 7 and 9; archie-core-afbk.4 adds 5 and 6 (verification, +// coverage/consistency); phase 8 (the merge gate) is archie-core-afbk.5. func PRReview() Workflow { return Workflow{ Name: "pr-review", @@ -82,6 +94,8 @@ func PRReview() Workflow { stagePRAnatomy(), stagePRLenses(), stagePRReview(), + stagePRVerification(), + stagePRCoverageConsistency(), stagePRSynthesis(), stagePROutput(), }, @@ -197,6 +211,12 @@ func stagePRAnatomy() Stage { return fmt.Errorf("compute blast radius: %w", err) } tc.prReview.blastRadius = blast + tc.prReview.clusters = prreview.ClusterFiles(tc.prReview.files) + exposure, err := prreview.ExposureCounts(os.DirFS(dir), changed) + if err != nil { + return fmt.Errorf("compute exposure counts: %w", err) + } + tc.prReview.highExposure = prreview.HighExposureFiles(exposure) mission := fmt.Sprintf( "Read this pull request's diff and enough of the surrounding repository to "+ @@ -349,31 +369,6 @@ func stagePRReview() Stage { // case a distinct StepFailed child step, so a reader of the execution tree // (not this slice) can tell "unreviewed" from "reviewed-clean" apart. func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) []prreview.Finding { - params := json.RawMessage(`{ - "type": "object", - "properties": { - "findings": { - "type": "array", - "items": { - "type": "object", - "properties": { - "file": {"type": "string"}, - "line_start": {"type": "integer"}, - "line_end": {"type": "integer"}, - "severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]}, - "title": {"type": "string"}, - "body": {"type": "string"}, - "suggestion": {"type": "string"}, - "evidence": {"type": "string"}, - "confidence": {"type": "number"}, - "tags": {"type": "array", "items": {"type": "string"}} - }, - "required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"] - } - } - }, - "required": ["findings"] - }`) mission := fmt.Sprintf( "%s\n\nTarget files: %s\n\nRead the target files (and, if useful, the context "+ "files) in the snapshot and report every finding for this dimension. Quote the "+ @@ -383,17 +378,63 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [ dim.Prompt, strings.Join(dim.TargetFiles, ", "), ) name := "reviewer-" + dim.Name - res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{{ - Name: "report_findings", Description: "Record this dimension's findings. Call exactly once, before finish.", - Parameters: params, RequiredFields: []string{"findings"}, MaxCalls: 1, - }}) + res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{reportFindingsTool}) if runErr != nil || res.Status != agentexec.StatusPassed { return nil } - calls := res.Captures["report_findings"] - if len(calls) != 1 { + findings, err := decodeReportedFindings(res.Captures["report_findings"], dim.Name) + if err != nil { return nil } + return findings +} + +// reportFindingsSchema is the finding shape every pr-review agent call that +// reports findings captures them in: phase 4's reviewer, phase 6's gap +// reviewers (which reuse runReviewer directly) and consistency verification. +var reportFindingsSchema = json.RawMessage(`{ + "type": "object", + "properties": { + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": {"type": "string"}, + "line_start": {"type": "integer"}, + "line_end": {"type": "integer"}, + "severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]}, + "title": {"type": "string"}, + "body": {"type": "string"}, + "suggestion": {"type": "string"}, + "evidence": {"type": "string"}, + "confidence": {"type": "number"}, + "tags": {"type": "array", "items": {"type": "string"}} + }, + "required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"] + } + } + }, + "required": ["findings"] +}`) + +// reportFindingsTool is the capture tool every findings-reporting call +// registers, at the shape reportFindingsSchema names. +var reportFindingsTool = agentexec.CaptureTool{ + Name: "report_findings", Description: "Record findings. Call exactly once, before finish.", + Parameters: reportFindingsSchema, RequiredFields: []string{"findings"}, MaxCalls: 1, +} + +// decodeReportedFindings decodes one report_findings capture into findings +// tagged with dimension, shared by every call site reportFindingsSchema +// backs. A call that never reported (no captures) or reported malformed JSON +// decodes to no findings and an error the caller treats as "nothing to add", +// not as a stage failure -- an agent that could not report is not evidence +// the code that follows should stop. +func decodeReportedFindings(calls []json.RawMessage, dimension string) ([]prreview.Finding, error) { + if len(calls) != 1 { + return nil, fmt.Errorf("report_findings called %d times (want exactly once)", len(calls)) + } var captured struct { Findings []struct { File string `json:"file"` @@ -409,7 +450,7 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [ } `json:"findings"` } if err := json.Unmarshal(calls[0], &captured); err != nil { - return nil + return nil, fmt.Errorf("decode findings: %w", err) } out := make([]prreview.Finding, len(captured.Findings)) for i, f := range captured.Findings { @@ -418,12 +459,12 @@ func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) [ lineEnd = f.LineStart } out[i] = prreview.Finding{ - Dimension: dim.Name, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd, + Dimension: dimension, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd, Severity: prreview.Severity(f.Severity), Title: f.Title, Body: f.Body, Suggestion: f.Suggestion, Evidence: f.Evidence, Confidence: f.Confidence, Tags: f.Tags, } } - return out + return out, nil } // prReviewMaxSteps bounds one reviewer's tool-loop iterations. Exhausting it @@ -475,10 +516,10 @@ func runReviewerAgent( // stagePRSynthesis is pipeline phase 7, code only: score, drop findings // under their severity's confidence floor, merge duplicates, rank, and cap -// at the inline comment limit. Phases 5, 6 and 8 (verification, coverage and -// consistency, the merge gate) do not run yet, so every surviving finding is -// scored on phase 4's raw output alone -- no compound, adversary or -// merge-gate multiplier has anything to apply yet. +// at the inline comment limit. Phase 8 (the merge gate) does not run yet, so +// every surviving finding's Blocking flag is whatever phase 5/6 left it as +// (false, since neither sets it -- that is archie-core-afbk.5's decision to +// make). func stagePRSynthesis() Stage { return Stage{Name: "synthesis", Run: func(_ context.Context, tc *TaskContext) error { scored := prreview.Score(tc.prReview.findings, prreview.ScoreInputs{ diff --git a/internal/domain/workflow/prreview_stages_verify.go b/internal/domain/workflow/prreview_stages_verify.go new file mode 100644 index 000000000..cb7f2f387 --- /dev/null +++ b/internal/domain/workflow/prreview_stages_verify.go @@ -0,0 +1,488 @@ +package workflow + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/domain/workflow/prreview" +) + +// stagePRVerification is pipeline phase 5, over the review findings phase 4 +// produced: extract each finding's evidence package (code), check the +// high-priority findings against theirs (an evidence verifier agent), have +// an adversary confirm or challenge every finding and add anything every +// reviewer missed, then flag within-cluster compound defects +// (docs/prds/pr-review-agent.md, phase 5). +func stagePRVerification() Stage { + return Stage{Name: "verification", Run: func(ctx context.Context, tc *TaskContext) error { + if len(tc.prReview.findings) == 0 { + return nil + } + + verified, err := runEvidenceVerifier(ctx, tc, tc.prReview.findings) + if err != nil { + return fmt.Errorf("evidence verifier: %w", err) + } + + confirmed, err := runAdversary(ctx, tc, verified) + if err != nil { + return fmt.Errorf("adversary: %w", err) + } + + compounded, err := runCompoundCheck(ctx, tc, confirmed) + if err != nil { + return fmt.Errorf("compound check: %w", err) + } + + tc.prReview.findings = compounded + return nil + }} +} + +// evidencePackageFor extracts a finding's evidence package from the +// snapshot, reusing afbk.1's ExtractEvidence. A file the snapshot no longer +// has (a finding pointing at a path that does not exist) extracts an empty +// package rather than failing the stage: that emptiness is itself the +// evidence verifier's answer. +func evidencePackageFor(tc *TaskContext, f prreview.Finding) prreview.EvidencePackage { + pkg, err := prreview.ExtractEvidence(os.DirFS(tc.prReview.snapshotDir), prreview.EvidenceRequest{ + File: f.File, LineStart: f.LineStart, LineEnd: f.LineEnd, + Title: f.Title, Body: f.Body, Evidence: f.Evidence, + }) + if err != nil { + return prreview.EvidencePackage{FindingTitle: f.Title, File: f.File} + } + return pkg +} + +// verifyFindingsSchema is the evidence verifier's capture-tool schema. +var verifyFindingsSchema = json.RawMessage(`{ + "type": "object", + "properties": { + "verdicts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "index": {"type": "integer"}, + "supported": {"type": "boolean"}, + "reason": {"type": "string"} + }, + "required": ["index", "supported"] + } + } + }, + "required": ["verdicts"] +}`) + +// evidenceCandidate is one high-priority finding the evidence verifier is +// asked to check, paired with its index in the caller's original slice: the +// verifier's answer references findings by that index, not by content. +type evidenceCandidate struct { + index int + finding prreview.Finding +} + +// highPriorityCandidates picks the findings runEvidenceVerifier checks: +// every high-priority finding, each still carrying its position in findings. +func highPriorityCandidates(findings []prreview.Finding) []evidenceCandidate { + var candidates []evidenceCandidate + for i, f := range findings { + if prreview.IsHighPriority(f) { + candidates = append(candidates, evidenceCandidate{index: i, finding: f}) + } + } + return candidates +} + +// evidenceListing renders each candidate's claim and evidence package as the +// evidence verifier's mission text. +func evidenceListing(tc *TaskContext, candidates []evidenceCandidate) string { + var listing strings.Builder + for _, c := range candidates { + pkg := evidencePackageFor(tc, c.finding) + fmt.Fprintf(&listing, "Finding %d: %s (%s:%d)\nClaim: %s\nQuoted evidence: %s\nCode at the cited lines:\n%s\n\n", + c.index, c.finding.Title, c.finding.File, c.finding.LineStart, c.finding.Body, c.finding.Evidence, pkg.PrimaryCode) + } + return listing.String() +} + +// runEvidenceVerifier runs phase 5's evidence verifier over every +// high-priority (critical or important) finding, dropping the ones it finds +// unsupported by their own evidence package. A finding below high priority is +// never sent, and always kept: the verifier exists to catch a hallucinated +// critical, not to grade a nitpick. A failed or malformed verifier call keeps +// every finding unfiltered -- the pipeline is recall-first, so a verifier +// that could not run is not evidence any finding is wrong; the error is +// intentionally swallowed here, not propagated. +func runEvidenceVerifier(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) { + candidates := highPriorityCandidates(findings) + if len(candidates) == 0 { + return findings, nil + } + + mission := fmt.Sprintf( + "Each finding below cites code as evidence for its claim. Check whether the "+ + "quoted code at the cited lines actually supports the claim -- not whether "+ + "the claim is a good catch, only whether the evidence is real and says what "+ + "the finding says it says.\n\n%s\n"+ + "Call verify_findings exactly once with one verdict per finding index, then "+ + "call finish with status \"passed\".", + evidenceListing(tc, candidates), + ) + res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "evidence-verifier", "review", mission, 15, []agentexec.CaptureTool{{ + Name: "verify_findings", Description: "Record each finding's evidence verdict. Call exactly once, before finish.", + Parameters: verifyFindingsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1, + }}) + if err != nil || res.Status != agentexec.StatusPassed { + return findings, nil //nolint:nilerr // recall-first: a verifier that could not run is not evidence any finding is wrong, so nothing is filtered + } + calls := res.Captures["verify_findings"] + if len(calls) != 1 { + return findings, nil + } + var captured struct { + Verdicts []struct { + Index int `json:"index"` + Supported bool `json:"supported"` + } `json:"verdicts"` + } + if err := json.Unmarshal(calls[0], &captured); err != nil { + return findings, nil //nolint:nilerr // malformed verifier output keeps findings unfiltered rather than failing the stage + } + unsupported := make(map[int]bool, len(captured.Verdicts)) + for _, v := range captured.Verdicts { + if !v.Supported { + unsupported[v.Index] = true + } + } + + kept := make([]prreview.Finding, 0, len(findings)) + for i, f := range findings { + if unsupported[i] { + continue + } + kept = append(kept, f) + } + return kept, nil +} + +// adversaryVerdictsSchema is the adversary's verdict capture-tool schema. +var adversaryVerdictsSchema = json.RawMessage(`{ + "type": "object", + "properties": { + "verdicts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "index": {"type": "integer"}, + "verdict": {"type": "string", "enum": ["confirmed", "challenged"]}, + "reason": {"type": "string"} + }, + "required": ["index", "verdict"] + } + } + }, + "required": ["verdicts"] +}`) + +// adversarySkepticism is the extra instruction the adversary's mission +// carries when intake scored the PR as likely machine-written, per the PRD. +func adversarySkepticism(aiGenerated float64) string { + if prreview.IsAIGenerated(aiGenerated) { + return "This PR's own description reads as likely machine-written: be more sceptical " + + "than usual, since a machine-written description is more likely to accompany " + + "machine-written code with the same failure modes." + } + return "This PR's own description does not read as machine-written; review normally." +} + +// findingsListing renders every finding, numbered by its index, for a +// mission that must reference findings back by that index. +func findingsListing(findings []prreview.Finding) string { + var listing strings.Builder + for i, f := range findings { + fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\nEvidence: %s\n\n", + i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence) + } + return listing.String() +} + +// applyAdversaryVerdicts decodes one adversary_verdicts capture and sets each +// referenced finding's Adversary field. An out-of-range index (malformed or +// adversarial model output) is skipped rather than panicking. +func applyAdversaryVerdicts(out []prreview.Finding, calls []json.RawMessage) { + if len(calls) != 1 { + return + } + var captured struct { + Verdicts []struct { + Index int `json:"index"` + Verdict string `json:"verdict"` + } `json:"verdicts"` + } + if json.Unmarshal(calls[0], &captured) != nil { + return + } + for _, v := range captured.Verdicts { + if v.Index < 0 || v.Index >= len(out) { + continue + } + switch v.Verdict { + case string(prreview.AdversaryConfirmed): + out[v.Index].Adversary = prreview.AdversaryConfirmed + case string(prreview.AdversaryChallenged): + out[v.Index].Adversary = prreview.AdversaryChallenged + } + } +} + +// runAdversary runs phase 5's adversary over every surviving finding at +// once: it confirms or challenges each by index, and may report new findings +// no reviewer caught. It reads intake's machine-written confidence and is +// told to be more sceptical above the threshold, per the PRD. A failed or +// malformed call leaves every finding's Adversary verdict unset (neither +// confirmed nor challenged) and adds nothing -- an adversary that could not +// run has reached no verdict, which is different from confirming everything. +func runAdversary(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) { + if len(findings) == 0 { + return findings, nil + } + + mission := fmt.Sprintf( + "You are the adversary reviewing this pull request's findings. For each finding "+ + "below, confirm it (you agree it is a real defect) or challenge it (you believe "+ + "it is wrong or not actually a problem). Read the snapshot yourself to check. "+ + "%s Also look for any defect every reviewer missed and report it as a new "+ + "finding.\n\n%s\n"+ + "Call adversary_verdicts exactly once with one verdict per finding index. If you "+ + "found anything new, call report_findings once too (skip it if you found "+ + "nothing new). Then call finish with status \"passed\".", + adversarySkepticism(tc.prReview.aiGenerated), findingsListing(findings), + ) + res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "adversary", "review", mission, 25, []agentexec.CaptureTool{ + { + Name: "adversary_verdicts", Description: "Record each finding's adversary verdict. Call exactly once, before finish.", + Parameters: adversaryVerdictsSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1, + }, + reportFindingsTool, + }) + if err != nil || res.Status != agentexec.StatusPassed { + return findings, nil //nolint:nilerr // a call that could not run reached no verdict, which the pipeline treats as neutral, not as a stage failure + } + + out := append([]prreview.Finding{}, findings...) + applyAdversaryVerdicts(out, res.Captures["adversary_verdicts"]) + if extra, err := decodeReportedFindings(res.Captures["report_findings"], "adversary"); err == nil { + out = append(out, extra...) + } + return out, nil +} + +// runCompoundCheck runs phase 5's compound-defect check once per cluster +// that has more than one finding, in parallel, bounded the same as every +// other pr-review fan-out. It flags the findings the agent names as +// compounding (Finding.Compound = true), which is what scoring's compound +// multiplier reads. A cluster with at most one finding cannot compound with +// anything and is skipped without an agent call. +func runCompoundCheck(ctx context.Context, tc *TaskContext, findings []prreview.Finding) ([]prreview.Finding, error) { + grouped := prreview.ClusterFindings(tc.prReview.clusters, findings) + var clusterIDs []string + for id, members := range grouped { + if len(members) > 1 { + clusterIDs = append(clusterIDs, id) + } + } + if len(clusterIDs) == 0 { + return findings, nil + } + + flagged := make([]map[string]bool, len(clusterIDs)) + forEachBounded(prReviewConcurrency, len(clusterIDs), func(i int) { + flagged[i] = runCompoundClusterCheck(ctx, tc, clusterIDs[i], grouped[clusterIDs[i]]) + }) + + compoundTitles := map[string]bool{} + for _, set := range flagged { + for title := range set { + compoundTitles[title] = true + } + } + out := append([]prreview.Finding{}, findings...) + for i, f := range out { + if compoundTitles[compoundKey(f)] { + out[i].Compound = true + } + } + return out, nil +} + +// compoundKey is how runCompoundCheck matches a cluster agent's flagged +// member back to a finding in the full list: findings carry no stable ID, so +// (file, line, title) is the identity a compound call's answer is matched on. +func compoundKey(f prreview.Finding) string { + return fmt.Sprintf("%s:%d:%s", f.File, f.LineStart, f.Title) +} + +// runCompoundClusterCheck runs one phase-5 compound call over one cluster's +// findings, returning the set of member keys (compoundKey) the agent flagged +// as only defective in combination. A failed or malformed call flags nothing. +func runCompoundClusterCheck(ctx context.Context, tc *TaskContext, clusterID string, members []prreview.Finding) map[string]bool { + var listing strings.Builder + for i, f := range members { + fmt.Fprintf(&listing, "Finding %d [%s] %s (%s:%d): %s\n\n", i, f.Severity, f.Title, f.File, f.LineStart, f.Body) + } + params := json.RawMessage(`{ + "type": "object", + "properties": { + "indices": {"type": "array", "items": {"type": "integer"}} + }, + "required": ["indices"] + }`) + mission := fmt.Sprintf( + "These findings all touch the same cluster of files. Read them together: is "+ + "there a defect that only exists because of how two or more of them combine -- "+ + "one that neither finding describes on its own? If so, call compound_findings "+ + "once with the indices of the findings that compound (an empty array if none "+ + "do). Then call finish with status \"passed\".\n\n%s", listing.String(), + ) + res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "compound-"+clusterID, "review", mission, 15, []agentexec.CaptureTool{{ + Name: "compound_findings", Description: "Record which findings compound. Call exactly once, before finish.", + Parameters: params, RequiredFields: []string{"indices"}, MaxCalls: 1, + }}) + if err != nil || res.Status != agentexec.StatusPassed { + return nil //nolint:nilerr // a cluster call that could not run flags nothing rather than failing the whole compound-check stage + } + calls := res.Captures["compound_findings"] + if len(calls) != 1 { + return nil + } + var captured struct { + Indices []int `json:"indices"` + } + if err := json.Unmarshal(calls[0], &captured); err != nil { + return nil + } + flagged := make(map[string]bool, len(captured.Indices)) + for _, index := range captured.Indices { + if index < 0 || index >= len(members) { + continue + } + flagged[compoundKey(members[index])] = true + } + return flagged +} + +// prReviewGapRounds is phase 6's cap on gap-review rounds: at most 2, per the +// PRD, so a coverage gap the gap reviewers cannot close does not loop forever. +const prReviewGapRounds = 2 + +// stagePRCoverageConsistency is pipeline phase 6: the coverage gate and +// consistency verification run in parallel and both append to +// tc.prReview.findings, guarded by prReviewState.findingsMu +// (docs/prds/pr-review-agent.md, phase 6: "Coverage and consistency, in +// parallel"). +func stagePRCoverageConsistency() Stage { + return Stage{Name: "coverage-consistency", Run: func(ctx context.Context, tc *TaskContext) error { + var wg sync.WaitGroup + var consistencyErr error + wg.Go(func() { + consistencyErr = runConsistencyVerification(ctx, tc) + }) + runCoverageGate(ctx, tc) + wg.Wait() + return consistencyErr + }} +} + +// runCoverageGate is phase 6's coverage gate: for up to prReviewGapRounds +// rounds, find the clusters and high-exposure blast-radius files no finding +// covers, run one gap reviewer per gap (reusing runReviewer, the same +// reviewer definition phase 4 uses), and append whatever they find. It stops +// early once nothing is uncovered, so a change that phase 4 already covered +// fully never spends a gap-review call. +func runCoverageGate(ctx context.Context, tc *TaskContext) { + for range prReviewGapRounds { + tc.prReview.findingsMu.Lock() + uncoveredClusters, uncoveredFiles := prreview.CoverageGaps(tc.prReview.clusters, tc.prReview.highExposure, tc.prReview.findings) + tc.prReview.findingsMu.Unlock() + if len(uncoveredClusters) == 0 && len(uncoveredFiles) == 0 { + return + } + + dims := gapDimensions(uncoveredClusters, uncoveredFiles) + found := make([][]prreview.Finding, len(dims)) + forEachBounded(prReviewConcurrency, len(dims), func(i int) { + found[i] = runReviewer(ctx, tc, dims[i]) + }) + + tc.prReview.findingsMu.Lock() + for _, f := range found { + tc.prReview.findings = append(tc.prReview.findings, f...) + } + tc.prReview.findingsMu.Unlock() + } +} + +// gapDimensions builds one reviewer dimension per coverage gap: a cluster's +// files together (a cluster is reviewed as one unit, the way phase 4 would +// have if a lens had proposed it), and one per uncovered high-exposure file. +func gapDimensions(clusters []prreview.Cluster, files []string) []prreview.Dimension { + dims := make([]prreview.Dimension, 0, len(clusters)+len(files)) + for _, cluster := range clusters { + dims = append(dims, prreview.Dimension{ + Name: "gap-" + cluster.ID, + Prompt: "This cluster of changed files was not reached by any review dimension. Review it for anything a reviewer would normally catch.", + TargetFiles: cluster.Files, + }) + } + for _, file := range files { + dims = append(dims, prreview.Dimension{ + Name: "gap-exposure-" + file, + Prompt: "This file is reached by more than one changed file (high blast-radius exposure) but no reviewer covered it. Review it for anything the changes upstream may have broken here.", + TargetFiles: []string{file}, + }) + } + return dims +} + +// runConsistencyVerification is phase 6's consistency verification: one +// agent call lists the cross-location obligations the changed code creates +// and verifies each by reading both ends, reporting only the broken ones as +// findings (the PRD's own wording -- "a broken obligation becomes a +// finding"). It reuses report_findings, the same shape every other +// findings-reporting call in the pipeline uses. +func runConsistencyVerification(ctx context.Context, tc *TaskContext) error { + mission := fmt.Sprintf( + "Read this pull request's diff and the surrounding repository. List every "+ + "cross-location obligation the changed code creates: what must be true "+ + "elsewhere in the codebase for a changed line to be correct (a caller that "+ + "must pass a new argument, a schema a struct must still match, a doc that must "+ + "still describe the behaviour, an invariant a comment states that the new code "+ + "must uphold). For each obligation, verify it by reading both ends yourself. "+ + "Report only the obligations you found broken as findings -- an obligation you "+ + "checked and found intact is not a finding.\n\nDiff:\n%s\n\n"+ + "Call report_findings once (an empty array if every obligation you checked "+ + "held), then call finish with status \"passed\".", + clip(tc.prReview.diff, 60000), + ) + res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "consistency", "review", mission, 25, []agentexec.CaptureTool{reportFindingsTool}) + if err != nil || res.Status != agentexec.StatusPassed { + return nil //nolint:nilerr // a consistency call that could not run reports no broken obligations rather than failing the stage + } + found, err := decodeReportedFindings(res.Captures["report_findings"], "consistency") + if err != nil { + return nil //nolint:nilerr // malformed consistency output reports no broken obligations rather than failing the stage + } + + tc.prReview.findingsMu.Lock() + tc.prReview.findings = append(tc.prReview.findings, found...) + tc.prReview.findingsMu.Unlock() + return nil +} diff --git a/internal/domain/workflow/prreview_stages_verify_test.go b/internal/domain/workflow/prreview_stages_verify_test.go new file mode 100644 index 000000000..7bcd0b86e --- /dev/null +++ b/internal/domain/workflow/prreview_stages_verify_test.go @@ -0,0 +1,195 @@ +package workflow + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/domain/workflow/prreview" +) + +// writeSnapshotFile is a small helper for tests that need a real file on disk +// for evidence extraction to read. +func writeSnapshotFile(t *testing.T, dir, path, content string) { + t.Helper() + full := filepath.Join(dir, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestStagePRVerificationDropsUnsupportedHighPriorityFindings(t *testing.T) { + tc := baseTaskContext(t) + dir := t.TempDir() + writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n") + tc.prReview = &prReviewState{ + snapshotDir: dir, + findings: []prreview.Finding{ + {File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityCritical, Title: "unsupported", Confidence: 0.9}, + {File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityNitpick, Title: "low priority", Confidence: 0.5}, + }, + } + tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) { + switch stage { + case "evidence-verifier": + return captureResult("verify_findings", map[string]any{ + "verdicts": []map[string]any{{"index": 0, "supported": false}}, + }), nil + case "adversary": + return captureResult("adversary_verdicts", map[string]any{"verdicts": []map[string]any{}}), nil + default: + return passedResult("ok"), nil + } + }} + + if err := stagePRVerification().Run(context.Background(), tc); err != nil { + t.Fatalf("verification: %v", err) + } + if len(tc.prReview.findings) != 1 { + t.Fatalf("findings = %d, want 1 (the unsupported critical finding dropped)", len(tc.prReview.findings)) + } + if tc.prReview.findings[0].Title != "low priority" { + t.Errorf("surviving finding = %q, want the low-priority one the verifier never checked", tc.prReview.findings[0].Title) + } +} + +func TestStagePRVerificationAppliesAdversaryVerdicts(t *testing.T) { + tc := baseTaskContext(t) + dir := t.TempDir() + writeSnapshotFile(t, dir, "a.go", "package a\n\nfunc F() int { return 1 }\n") + tc.prReview = &prReviewState{ + snapshotDir: dir, + findings: []prreview.Finding{ + {File: "a.go", LineStart: 3, LineEnd: 3, Severity: prreview.SeverityImportant, Title: "f0", Confidence: 0.9}, + }, + } + tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) { + switch stage { + case "evidence-verifier": + return captureResult("verify_findings", map[string]any{ + "verdicts": []map[string]any{{"index": 0, "supported": true}}, + }), nil + case "adversary": + return captureResult("adversary_verdicts", map[string]any{ + "verdicts": []map[string]any{{"index": 0, "verdict": "confirmed"}}, + }), nil + default: + return passedResult("ok"), nil + } + }} + + if err := stagePRVerification().Run(context.Background(), tc); err != nil { + t.Fatalf("verification: %v", err) + } + if len(tc.prReview.findings) != 1 { + t.Fatalf("findings = %d, want 1", len(tc.prReview.findings)) + } + if tc.prReview.findings[0].Adversary != prreview.AdversaryConfirmed { + t.Errorf("adversary verdict = %q, want confirmed", tc.prReview.findings[0].Adversary) + } +} + +func TestStagePRVerificationRunsNoAgentCallsWhenNoFindings(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{snapshotDir: t.TempDir()} + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + t.Fatal("no agent call expected with zero findings") + return agentexec.Result{}, nil + }} + if err := stagePRVerification().Run(context.Background(), tc); err != nil { + t.Fatalf("verification: %v", err) + } +} + +func TestStagePRCompoundMarksFlaggedFindingsWithinCluster(t *testing.T) { + tc := baseTaskContext(t) + dir := t.TempDir() + tc.prReview = &prReviewState{ + snapshotDir: dir, + clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go", "b.go"}}}, + findings: []prreview.Finding{ + {File: "a.go", LineStart: 1, Title: "f0", Severity: prreview.SeverityImportant, Confidence: 0.9}, + {File: "b.go", LineStart: 1, Title: "f1", Severity: prreview.SeverityImportant, Confidence: 0.9}, + }, + } + tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) { + if stage == "compound-cluster_0" { + return captureResult("compound_findings", map[string]any{"indices": []int{0, 1}}), nil + } + return passedResult("ok"), nil + }} + + findings, err := runCompoundCheck(context.Background(), tc, tc.prReview.findings) + if err != nil { + t.Fatalf("runCompoundCheck: %v", err) + } + for _, f := range findings { + if !f.Compound { + t.Errorf("finding %q Compound = false, want true (both findings in the flagged cluster)", f.Title) + } + } +} + +func TestStagePRCoverageConsistencyRunsGapReviewersAndConsistencyInParallel(t *testing.T) { + tc := baseTaskContext(t) + dir := t.TempDir() + tc.prReview = &prReviewState{ + snapshotDir: dir, + clusters: []prreview.Cluster{{ID: "cluster_0", Name: "pkg", Files: []string{"pkg/a.go"}}}, + highExposure: []string{"pkg/shared.go"}, + findings: nil, + } + tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) { + switch stage { + case "consistency": + return captureResult("report_findings", map[string]any{ + "findings": []map[string]any{ + {"file": "pkg/a.go", "line_start": 1, "severity": "important", "title": "broken obligation", "body": "b", "evidence": "e", "confidence": 0.8}, + }, + }), nil + default: + // Gap reviewers reuse runReviewer's report_findings shape. + return captureResult("report_findings", map[string]any{"findings": []any{}}), nil + } + }} + + if err := stagePRCoverageConsistency().Run(context.Background(), tc); err != nil { + t.Fatalf("coverage-consistency: %v", err) + } + + foundConsistency := false + for _, f := range tc.prReview.findings { + if f.Title == "broken obligation" { + foundConsistency = true + } + } + if !foundConsistency { + t.Error("consistency verification's finding was not appended") + } +} + +func TestCoverageGapCapsAtTwoRounds(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{ + snapshotDir: t.TempDir(), + clusters: []prreview.Cluster{{ID: "cluster_0", Files: []string{"a.go"}}}, + } + var calls int + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + calls++ + // Every gap reviewer call reports nothing, so the cluster stays + // uncovered and a naive loop would run forever without the cap. + return captureResult("report_findings", map[string]any{"findings": []any{}}), nil + }} + + runCoverageGate(context.Background(), tc) + + if calls != 2 { + t.Errorf("gap reviewer calls = %d, want exactly 2 (the round cap)", calls) + } +}