diff --git a/internal/domain/workflow/definition.go b/internal/domain/workflow/definition.go index 9b5e81177..b90c3809c 100644 --- a/internal/domain/workflow/definition.go +++ b/internal/domain/workflow/definition.go @@ -202,5 +202,6 @@ func legacyBuiltinWorkflows() Registry { return Registry{ "bootstrap": Bootstrap(), "implement": Implement(), "tdd": TDD(), "feasibility": Feasibility(), "triage": Triage(), "remediate": Remediate(), + "pr-review": PRReview(), } } diff --git a/internal/domain/workflow/outcome_sites_test.go b/internal/domain/workflow/outcome_sites_test.go index e80704a05..3fe5bc5a3 100644 --- a/internal/domain/workflow/outcome_sites_test.go +++ b/internal/domain/workflow/outcome_sites_test.go @@ -41,6 +41,7 @@ var outcomeSites = []outcomeSite{ {"diff_rules.go", "StatusParked", "the diff breaks one of the repo's blocking rules"}, {"feasibility.go", "StatusClosedWontDo", "feasibility decided this work should not be done"}, {"feasibility.go", "StatusWaitingHuman", "the PRD is delivered and the run waits for a go/no-go"}, + {"prreview_stages.go", "StatusCompleted", "the pr-review pipeline posted its findings (or found none to post)"}, {"remediate.go", "StatusParked", "the remediation round cap is spent"}, {"remediate.go", "StatusPROpen", "a remediation round pushed the task back to its open PR"}, {"review.go", "StatusParked", "review is enabled but no reviewer is wired"}, diff --git a/internal/domain/workflow/prreview_stages.go b/internal/domain/workflow/prreview_stages.go new file mode 100644 index 000000000..92089bbfd --- /dev/null +++ b/internal/domain/workflow/prreview_stages.go @@ -0,0 +1,648 @@ +package workflow + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "sync" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/domain/workflow/prreview" + "github.com/samcharles93/archie-core/internal/domain/workflow/task" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// PRSource fetches a pull request under review: its descriptive content, its +// diff, and a read-only, .git-free snapshot of its head. It is Trees' +// counterpart for the pr-review workflow -- Trees reaches the task's own +// worktree, which the pipeline must never read (docs/prds/pr-review-agent.md, +// Isolation); PRSource reaches the arbitrary pull request named by +// Task.Owner/Repo/PRNumber instead, and carries no push credential. +// +// The real forge-backed implementation belongs to whichever bead wires +// pr-review's triggers (archie-core-afbk.7); this bead builds and tests the +// Stage wiring against a fake. +type PRSource interface { + // Metadata returns the pull request's title, body and commit messages -- + // phase 1's "PR metadata" and phase 3's hallucination-check dimension's + // "what the PR claims". + Metadata(ctx context.Context, owner, repo string, number int) (PRMetadata, error) + // Diff returns the pull request's unified diff against its base branch. + Diff(ctx context.Context, owner, repo string, number int) (string, error) + // Snapshot materializes the pull request head's tracked files into + // destDir with no .git directory present, and returns the head commit + // SHA those files (and the diff's line numbers) were measured on. + Snapshot(ctx context.Context, owner, repo string, number int, destDir string) (headSHA string, err error) +} + +// PRMetadata is a pull request's descriptive content. +type PRMetadata struct { + Title string + Body string + Commits []string // commit messages, oldest first +} + +// prReviewState is the pr-review workflow's cross-stage scratch state: each +// phase's output for the phases after it to consume. Unexported, following +// the pattern the feasibility workflow's decision field set. +type prReviewState struct { + metadata PRMetadata + diff string + files []prreview.FileChange + stats prreview.DiffStats + depth prreview.Depth + aiGenerated float64 + + snapshotDir string + headSHA string + blastRadius []string + + narrative string + dimensions []prreview.Dimension + findings []prreview.Finding + scored []prreview.ScoredFinding +} + +// prReviewConcurrency bounds phases 3 (lenses), 4 (reviewers) and 9 (polish), +// all "in parallel, at most 8 at a time" per the PRD except phase 3, which +// never has more than three lenses to begin with. +const prReviewConcurrency = 8 + +// PRReview is the pull request reviewer: a fixed pipeline, not a single +// agent (docs/prds/pr-review-agent.md). This bead wires phases 1, 2, 3, 4, 7 +// and 9; phases 5, 6 and 8 (verification, coverage/consistency, merge gate) +// are archie-core-afbk.4 and .5. +func PRReview() Workflow { + return Workflow{ + Name: "pr-review", + Stages: []Stage{ + stagePRIntake(), + stagePRAnatomy(), + stagePRLenses(), + stagePRReview(), + stagePRSynthesis(), + stagePROutput(), + }, + } +} + +// stagePRIntake is pipeline phase 1: PR metadata and diff statistics, the +// depth they classify, and the machine-written confidence phase 3's +// hallucination-check dimension and phase 7's scoring both read. +func stagePRIntake() Stage { + return Stage{Name: "intake", Run: func(ctx context.Context, tc *TaskContext) error { + if tc.PRSource == nil { + return fmt.Errorf("pr-review: no PRSource configured") + } + meta, err := tc.PRSource.Metadata(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber) + if err != nil { + return fmt.Errorf("fetch pull request metadata: %w", err) + } + diff, err := tc.PRSource.Diff(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber) + if err != nil { + return fmt.Errorf("fetch pull request diff: %w", err) + } + files := prreview.ParseDiff(diff) + stats := prreview.SummarizeFiles(files) + depth := prreview.ResolveDepth(stats.TotalAdditions+stats.TotalDeletions, "") + + aiGenerated, err := scoreAIGenerated(ctx, tc, meta) + if err != nil { + return fmt.Errorf("score likely machine-written PR: %w", err) + } + + tc.prReview = &prReviewState{ + metadata: meta, diff: diff, files: files, stats: stats, + depth: depth, aiGenerated: aiGenerated, + } + return nil + }} +} + +// scoreAIGenerated runs phase 1's classification-role call: how confident an +// agent is that the PR's own description reads as machine-written, which +// phase 3's hallucination-check dimension and phase 7's scoring both use. +func scoreAIGenerated(ctx context.Context, tc *TaskContext, meta PRMetadata) (float64, error) { + // This call reads no files -- only the metadata already in its mission -- + // so it gets its own empty scratch directory rather than the pull + // request's snapshot, which does not exist yet at phase 1. + scratch, err := os.MkdirTemp("", "pr-review-intake-*") + if err != nil { + return 0, fmt.Errorf("create intake scratch directory: %w", err) + } + defer os.RemoveAll(scratch) + + params := json.RawMessage(`{ + "type": "object", + "properties": { + "confidence": {"type": "number", "description": "0 = clearly written by the human author, 1 = clearly machine-generated."}, + "reasons": {"type": "string", "description": "What in the title, body or commit messages drove the confidence."} + }, + "required": ["confidence"] + }`) + mission := fmt.Sprintf( + "Judge whether this pull request's own description reads as written by an AI "+ + "coding assistant rather than the human who opened it: generic boilerplate, a "+ + "summary that oversells the diff, or phrasing typical of an LLM's own commit "+ + "or PR style. This is about the PR's description, not the code.\n\n"+ + "Title: %s\n\nDescription:\n%s\n\nCommit messages:\n%s\n\n"+ + "Call score_ai_generated exactly once with your confidence, then call finish "+ + "with status \"passed\".", + meta.Title, meta.Body, strings.Join(meta.Commits, "\n"), + ) + res, err := runPRReviewAgent(ctx, tc, scratch, "intake-ai-score", "classification", mission, 6, []agentexec.CaptureTool{{ + Name: "score_ai_generated", Description: "Record the machine-written confidence. Call exactly once, before finish.", + Parameters: params, RequiredFields: []string{"confidence"}, MaxCalls: 1, + }}) + if err != nil { + return 0, err + } + calls := res.Captures["score_ai_generated"] + if len(calls) != 1 { + return 0, fmt.Errorf("intake-ai-score called score_ai_generated %d times (want exactly once)", len(calls)) + } + var captured struct { + Confidence float64 `json:"confidence"` + } + if err := json.Unmarshal(calls[0], &captured); err != nil { + return 0, fmt.Errorf("decode machine-written confidence: %w", err) + } + return captured.Confidence, nil +} + +// stagePRAnatomy is pipeline phase 2: the read-only snapshot every later +// phase reads from, the deterministic clustering and blast radius, and one +// agent's narrative of the change. +func stagePRAnatomy() Stage { + return Stage{Name: "anatomy", Run: func(ctx context.Context, tc *TaskContext) error { + dir, err := os.MkdirTemp("", fmt.Sprintf("pr-review-snapshot-task%d-*", tc.Task.ID)) + if err != nil { + return fmt.Errorf("create pull request snapshot directory: %w", err) + } + headSHA, err := tc.PRSource.Snapshot(ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber, dir) + if err != nil { + return fmt.Errorf("snapshot pull request head: %w", err) + } + tc.prReview.snapshotDir = dir + tc.prReview.headSHA = headSHA + + changed := make([]string, 0, len(tc.prReview.files)) + for _, f := range tc.prReview.files { + changed = append(changed, f.Path) + } + blast, err := prreview.BlastRadius(os.DirFS(dir), changed) + if err != nil { + return fmt.Errorf("compute blast radius: %w", err) + } + tc.prReview.blastRadius = blast + + mission := fmt.Sprintf( + "Read this pull request's diff and enough of the surrounding repository to "+ + "write a short narrative of the change: what it does, the riskiest surfaces "+ + "it touches, any change that does not fit the PR's own description, and any "+ + "gap between what the description claims and what the diff carries out.\n\n"+ + "Title: %s\n\nDescription:\n%s\n\nDiff:\n%s\n\n"+ + "Call finish with status \"passed\" and your narrative as the summary.", + tc.prReview.metadata.Title, tc.prReview.metadata.Body, clip(tc.prReview.diff, 60000), + ) + res, err := runPRReviewAgent(ctx, tc, tc.prReview.snapshotDir, "anatomy", "review", mission, 15, nil) + if err != nil { + return err + } + tc.prReview.narrative = res.Summary + return nil + }} +} + +// prReviewLenses are phase 3's three fixed lenses. Each proposes review +// dimensions from a different angle; code merges, dedups and caps the result. +var prReviewLenses = []struct { + name string + prompt string +}{ + {name: "behaviour", prompt: "where the old and new code diverge for any input: API contracts, concurrency and state, security, error handling, data flow"}, + {name: "mechanics", prompt: "whether the code runs at the language and framework level: types, signatures and every caller, decorators and middleware, framework contracts, imports"}, + {name: "fit", prompt: "patterns, complexity, abstraction, test adequacy, documentation that no longer matches, dependencies, migration completeness"}, +} + +// stagePRLenses is pipeline phase 3: three lens agents proposing review +// dimensions in parallel, merged with the hallucination-check dimension (for +// a likely machine-written PR) as one more candidate, deduplicated and +// capped by code. The hallucination dimension competes for the same cap as +// everything else -- "at most N dimensions" is a total, not a total plus one +// -- so it is added before MergeDimensions cuts the tail, never after. +func stagePRLenses() Stage { + return Stage{Name: "lenses", Run: func(ctx context.Context, tc *TaskContext) error { + outputs := make([][]prreview.Dimension, len(prReviewLenses)) + errs := make([]error, len(prReviewLenses)) + forEachBounded(len(prReviewLenses), len(prReviewLenses), func(i int) { + outputs[i], errs[i] = runLens(ctx, tc, prReviewLenses[i].name, prReviewLenses[i].prompt) + }) + for i, err := range errs { + if err != nil { + return fmt.Errorf("lens %s: %w", prReviewLenses[i].name, err) + } + } + candidates := outputs + if extra := prreview.HallucinationDimension(tc.prReview.aiGenerated); extra != nil { + candidates = append(append([][]prreview.Dimension{}, outputs...), []prreview.Dimension{*extra}) + } + tc.prReview.dimensions = prreview.MergeDimensions(candidates, prreview.MaxDimensionsFor(tc.prReview.depth)) + return nil + }} +} + +// runLens runs one phase-3 lens call and decodes its proposed dimensions. +func runLens(ctx context.Context, tc *TaskContext, name, angle string) ([]prreview.Dimension, error) { + params := json.RawMessage(`{ + "type": "object", + "properties": { + "dimensions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "prompt": {"type": "string", "description": "The reviewer's mission for this dimension, written for this PR."}, + "target_files": {"type": "array", "items": {"type": "string"}}, + "context_files": {"type": "array", "items": {"type": "string"}}, + "priority": {"type": "number"} + }, + "required": ["name", "prompt", "target_files", "priority"] + } + } + }, + "required": ["dimensions"] + }`) + mission := fmt.Sprintf( + "You are the %s lens over this pull request: %s.\n\n"+ + "Title: %s\n\nDescription:\n%s\n\nDiff:\n%s\n\n"+ + "Propose the review dimensions this angle needs, each with a reviewer prompt "+ + "written for this specific PR, its target files, any context files a reviewer "+ + "should also read, and a priority (higher runs first when the depth caps how "+ + "many dimensions survive). Call propose_dimensions exactly once, then call "+ + "finish with status \"passed\".", + name, angle, tc.prReview.metadata.Title, tc.prReview.metadata.Body, clip(tc.prReview.diff, 60000), + ) + res, err := runPRReviewAgent(ctx, tc, tc.prReview.snapshotDir, "lens-"+name, "review", mission, 15, []agentexec.CaptureTool{{ + Name: "propose_dimensions", Description: "Record this lens's proposed review dimensions. Call exactly once, before finish.", + Parameters: params, RequiredFields: []string{"dimensions"}, MaxCalls: 1, + }}) + if err != nil { + return nil, err + } + calls := res.Captures["propose_dimensions"] + if len(calls) != 1 { + return nil, fmt.Errorf("lens %s called propose_dimensions %d times (want exactly once)", name, len(calls)) + } + var captured struct { + Dimensions []struct { + Name string `json:"name"` + Prompt string `json:"prompt"` + TargetFiles []string `json:"target_files"` + ContextFiles []string `json:"context_files"` + Priority float64 `json:"priority"` + } `json:"dimensions"` + } + if err := json.Unmarshal(calls[0], &captured); err != nil { + return nil, fmt.Errorf("decode lens %s dimensions: %w", name, err) + } + out := make([]prreview.Dimension, len(captured.Dimensions)) + for i, d := range captured.Dimensions { + out[i] = prreview.Dimension{ + Name: d.Name, Prompt: d.Prompt, TargetFiles: d.TargetFiles, + ContextFiles: d.ContextFiles, Priority: d.Priority, + } + } + return out, nil +} + +// stagePRReview is pipeline phase 4: one reviewer call per dimension, at most +// prReviewConcurrency at a time. A reviewer that exhausts its turn cap or +// deadline before its terminal tool call records StepFailed -- distinct from +// StepSucceeded for a reviewer that completed and found nothing -- so +// synthesis and the posted review can tell "unreviewed" from "reviewed-clean" +// (docs/prds/pr-review-agent.md, phase 4). +func stagePRReview() Stage { + return Stage{Name: "review", Run: func(ctx context.Context, tc *TaskContext) error { + dimensions := tc.prReview.dimensions + findings := make([][]prreview.Finding, len(dimensions)) + forEachBounded(prReviewConcurrency, len(dimensions), func(i int) { + findings[i] = runReviewer(ctx, tc, dimensions[i]) + }) + var all []prreview.Finding + for _, f := range findings { + all = append(all, f...) + } + tc.prReview.findings = all + return nil + }} +} + +// runReviewer runs one phase-4 reviewer call. It never returns a Go error to +// its caller: a reviewer that failed, or did not reach its terminal tool +// call, contributes no findings -- the same zero-findings shape a reviewer +// that ran cleanly and found nothing produces. The two are not confused in +// the record, only in this return value: runReviewerAgent gives the first +// case a distinct StepFailed child step, so a reader of the execution tree +// (not this slice) can tell "unreviewed" from "reviewed-clean" apart. +func runReviewer(ctx context.Context, tc *TaskContext, dim prreview.Dimension) []prreview.Finding { + params := json.RawMessage(`{ + "type": "object", + "properties": { + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file": {"type": "string"}, + "line_start": {"type": "integer"}, + "line_end": {"type": "integer"}, + "severity": {"type": "string", "enum": ["critical", "important", "suggestion", "nitpick"]}, + "title": {"type": "string"}, + "body": {"type": "string"}, + "suggestion": {"type": "string"}, + "evidence": {"type": "string"}, + "confidence": {"type": "number"}, + "tags": {"type": "array", "items": {"type": "string"}} + }, + "required": ["file", "line_start", "severity", "title", "body", "evidence", "confidence"] + } + } + }, + "required": ["findings"] + }`) + mission := fmt.Sprintf( + "%s\n\nTarget files: %s\n\nRead the target files (and, if useful, the context "+ + "files) in the snapshot and report every finding for this dimension. Quote the "+ + "exact evidence for each finding; do not report anything you cannot point at in "+ + "the code. Call report_findings exactly once (an empty array is a valid, "+ + "complete report), then call finish with status \"passed\".", + dim.Prompt, strings.Join(dim.TargetFiles, ", "), + ) + name := "reviewer-" + dim.Name + res, runErr := runReviewerAgent(ctx, tc, name, mission, []agentexec.CaptureTool{{ + Name: "report_findings", Description: "Record this dimension's findings. Call exactly once, before finish.", + Parameters: params, RequiredFields: []string{"findings"}, MaxCalls: 1, + }}) + if runErr != nil || res.Status != agentexec.StatusPassed { + return nil + } + calls := res.Captures["report_findings"] + if len(calls) != 1 { + return nil + } + var captured struct { + Findings []struct { + File string `json:"file"` + LineStart int `json:"line_start"` + LineEnd int `json:"line_end"` + Severity string `json:"severity"` + Title string `json:"title"` + Body string `json:"body"` + Suggestion string `json:"suggestion"` + Evidence string `json:"evidence"` + Confidence float64 `json:"confidence"` + Tags []string `json:"tags"` + } `json:"findings"` + } + if err := json.Unmarshal(calls[0], &captured); err != nil { + return nil + } + out := make([]prreview.Finding, len(captured.Findings)) + for i, f := range captured.Findings { + lineEnd := f.LineEnd + if lineEnd == 0 { + lineEnd = f.LineStart + } + out[i] = prreview.Finding{ + Dimension: dim.Name, File: f.File, LineStart: f.LineStart, LineEnd: lineEnd, + Severity: prreview.Severity(f.Severity), Title: f.Title, Body: f.Body, + Suggestion: f.Suggestion, Evidence: f.Evidence, Confidence: f.Confidence, Tags: f.Tags, + } + } + return out +} + +// prReviewMaxSteps bounds one reviewer's tool-loop iterations. Exhausting it +// (or the wall-clock budget) before the terminal finish call is exactly the +// "turn cap or deadline" case the phase-4 status distinction exists for. +const prReviewMaxSteps = 25 + +// runReviewerAgent runs one phase-4 reviewer call and records its child step +// with the phase-4 status distinction (docs/prds/pr-review-agent.md, phase +// 4): StepFailed for a reviewer that exhausted its turn cap or deadline +// before its terminal tool call, StepSucceeded for one that reached it -- +// whether or not it reported any findings. tc.RunAgentChild does not serve +// this: it maps only the Go error, so a "parked" (turn-cap/deadline) result +// with no Go error would record StepSucceeded exactly like a clean, +// zero-findings run, and the two would be indistinguishable to anything +// reading the execution tree afterwards. +func runReviewerAgent( + ctx context.Context, tc *TaskContext, name, mission string, captureTools []agentexec.CaptureTool, +) (agentexec.Result, error) { + modelRef := tc.Cfg.Models["review"] + if modelRef == "" { + return agentexec.Result{}, fmt.Errorf("no model configured for role %q (set [models] in config)", "review") + } + req := agentexec.Request{ + Version: agentexec.ProtocolVersion, TaskID: tc.Task.ID, Attempt: tc.Task.Attempt, + Stage: name, Workflow: tc.Task.Workflow, Model: modelRef, + ContextWindow: modelContextBudget(tc.Cfg, modelRef), + Mission: mission, ReadOnly: true, + Budget: agentexec.Budget{MaxSteps: prReviewMaxSteps, WallClock: tc.Cfg.Budgets.WallClock.Std()}, + CaptureTools: captureTools, + } + stepID, _, err := tc.startChildStep(ctx, task.StepKindAgent, name) + if err != nil { + return agentexec.Result{}, err + } + res, runErr := tc.Agent.Run(ctx, tc.prReview.snapshotDir, req, tc.toolCallReporter(name)) + to, detail := taskstate.StepSucceeded, res.Summary + switch { + case runErr != nil: + to, detail = taskstate.StepFailed, runErr.Error() + case res.Status != agentexec.StatusPassed: + to, detail = taskstate.StepFailed, fmt.Sprintf("reviewer did not complete (%s: %s)", res.Status, res.StopReason) + } + if ferr := tc.finishChildStep(ctx, stepID, to, detail, int64(res.TokensUsed)); ferr != nil { + return res, ferr + } + return res, runErr +} + +// stagePRSynthesis is pipeline phase 7, code only: score, drop findings +// under their severity's confidence floor, merge duplicates, rank, and cap +// at the inline comment limit. Phases 5, 6 and 8 (verification, coverage and +// consistency, the merge gate) do not run yet, so every surviving finding is +// scored on phase 4's raw output alone -- no compound, adversary or +// merge-gate multiplier has anything to apply yet. +func stagePRSynthesis() Stage { + return Stage{Name: "synthesis", Run: func(_ context.Context, tc *TaskContext) error { + scored := prreview.Score(tc.prReview.findings, prreview.ScoreInputs{ + AIGenerated: tc.prReview.aiGenerated, + BlastRadiusFiles: len(tc.prReview.blastRadius), + }) + tc.prReview.scored = prreview.CapInlineComments(scored) + return nil + }} +} + +// stagePROutput is pipeline phase 9: one polish call per comment, at most +// prReviewConcurrency at a time, keeping the original wording on failure, +// then posting the surviving inline-anchored findings as one forge review. +// The merge gate (phase 8) does not run yet, so there is no blocking +// classification to decide REQUEST_CHANGES; every posted review is a +// comment until archie-core-afbk.5 lands that decision. +func stagePROutput() Stage { + return Stage{Name: "output", Run: func(ctx context.Context, tc *TaskContext) error { + polished := make([]prreview.ScoredFinding, len(tc.prReview.scored)) + forEachBounded(prReviewConcurrency, len(tc.prReview.scored), func(i int) { + polished[i] = polishFinding(ctx, tc, tc.prReview.scored[i]) + }) + tc.prReview.scored = polished + + if err := postPRReview(ctx, tc); err != nil { + return fmt.Errorf("post review: %w", err) + } + tc.Outcome = Outcome{ + Status: StatusCompleted, + Detail: fmt.Sprintf("posted %d finding(s)", len(polished)), + } + return nil + }} +} + +// polishFinding runs one phase-9 polish call, tightening a finding's wording. +// It keeps the original finding unchanged on any failure, per the PRD. +func polishFinding(ctx context.Context, tc *TaskContext, f prreview.ScoredFinding) prreview.ScoredFinding { + params := json.RawMessage(`{ + "type": "object", + "properties": {"body": {"type": "string", "description": "The tightened finding body."}}, + "required": ["body"] + }`) + mission := fmt.Sprintf( + "Tighten the wording of this review comment without changing its meaning or "+ + "dropping any concrete detail (file, line, failure scenario).\n\n"+ + "Title: %s\nBody: %s\n\nCall polish exactly once with the tightened body, then "+ + "call finish with status \"passed\".", + f.Title, f.Body, + ) + res, err := runPRReviewAgent(ctx, tc, tc.prReview.snapshotDir, "polish", "classification", mission, 4, []agentexec.CaptureTool{{ + Name: "polish", Description: "Record the tightened comment body. Call exactly once, before finish.", + Parameters: params, RequiredFields: []string{"body"}, MaxCalls: 1, + }}) + if err != nil { + return f + } + calls := res.Captures["polish"] + if len(calls) != 1 { + return f + } + var captured struct { + Body string `json:"body"` + } + if err := json.Unmarshal(calls[0], &captured); err != nil || captured.Body == "" { + return f + } + f.Body = captured.Body + return f +} + +// postPRReview posts every line-anchored finding as one forge review. A +// finding with no line (LineStart <= 0) has nothing to post yet: the PR-body +// fallback list is the merge gate stage's concern (archie-core-afbk.5), not +// this bead's. +func postPRReview(ctx context.Context, tc *TaskContext) error { + comments := make([]ReviewComment, 0, len(tc.prReview.scored)) + for _, f := range tc.prReview.scored { + if f.LineStart <= 0 { + continue + } + comments = append(comments, ReviewComment{Path: f.File, Line: f.LineStart, Body: f.Body}) + } + if len(comments) == 0 { + return nil + } + return tc.Forge.CreateReviewComments( + ctx, tc.Task.Owner, tc.Task.Repo, tc.Task.PRNumber, tc.prReview.headSHA, comments, + ) +} + +// runPRReviewAgent runs one read-only agent call rooted at workspace -- never +// tc.Dir, per Isolation -- and discards the run/park distinction: any +// non-passed result is a stage error. Every pr-review phase but the reviewer +// fan-out (which must tell "did not finish" from "finished, found nothing" +// apart) uses this. Every phase from anatomy onward passes +// tc.prReview.snapshotDir; intake's AI-score call runs before that snapshot +// exists, so it passes its own scratch workspace instead. +func runPRReviewAgent( + ctx context.Context, tc *TaskContext, workspace, name, role, mission string, maxSteps int, captureTools []agentexec.CaptureTool, +) (agentexec.Result, error) { + res, err := runPRReviewAgentRecorded(ctx, tc, workspace, name, role, mission, maxSteps, captureTools) + if err != nil { + return res, err + } + if res.Status != agentexec.StatusPassed { + detail := res.Detail + if detail == "" { + detail = res.Summary + } + return res, fmt.Errorf("%s %s (%s): %s", name, res.Status, res.StopReason, clip(detail, 2000)) + } + return res, nil +} + +// runPRReviewAgentRecorded is the shared request-building and StepExecution +// recording underneath every pr-review agent call. It returns whatever the +// runtime returned, status and all: callers that must distinguish "did not +// finish" from "finished, found nothing" (the reviewer fan-out) read +// res.Status themselves instead of getting an error for anything but a Go +// error. +func runPRReviewAgentRecorded( + ctx context.Context, tc *TaskContext, workspace, name, role, mission string, maxSteps int, captureTools []agentexec.CaptureTool, +) (agentexec.Result, error) { + modelRef := tc.Cfg.Models[role] + if modelRef == "" { + return agentexec.Result{}, fmt.Errorf("no model configured for role %q (set [models] in config)", role) + } + req := agentexec.Request{ + Version: agentexec.ProtocolVersion, TaskID: tc.Task.ID, Attempt: tc.Task.Attempt, + Stage: name, Workflow: tc.Task.Workflow, Model: modelRef, + ContextWindow: modelContextBudget(tc.Cfg, modelRef), + Mission: mission, ReadOnly: true, + Budget: agentexec.Budget{MaxSteps: maxSteps, WallClock: tc.Cfg.Budgets.WallClock.Std()}, + CaptureTools: captureTools, + } + return tc.RunAgentChild(ctx, name, func() (agentexec.Result, error) { + return tc.Agent.Run(ctx, workspace, req, tc.toolCallReporter(name)) + }) +} + +// forEachBounded runs fn(0), fn(1), ..., fn(n-1), at most limit at a time, +// and waits for all of them. It is phase 3, 4 and 9's shared concurrency +// bound: tc.RunAgentChild already supports concurrent fan-out from inside +// one stage body (each call independently records a StepExecution child), so +// this needs only a semaphore, not an engine change. +func forEachBounded(limit, n int, fn func(i int)) { + if n == 0 { + return + } + if limit <= 0 || limit > n { + limit = n + } + sem := make(chan struct{}, limit) + var wg sync.WaitGroup + for i := range n { + wg.Add(1) + sem <- struct{}{} + go func(i int) { + defer wg.Done() + defer func() { <-sem }() + fn(i) + }(i) + } + wg.Wait() +} diff --git a/internal/domain/workflow/prreview_stages_test.go b/internal/domain/workflow/prreview_stages_test.go new file mode 100644 index 000000000..e3bc9a0d9 --- /dev/null +++ b/internal/domain/workflow/prreview_stages_test.go @@ -0,0 +1,408 @@ +package workflow + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "sync" + "sync/atomic" + "testing" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/workflow/prreview" + "github.com/samcharles93/archie-core/internal/events" + "github.com/samcharles93/archie-core/internal/taskstate" +) + +// fakePRSource is a test double for PRSource: no forge call, just canned +// metadata, diff and a snapshot copied from an in-memory file set. +type fakePRSource struct { + metadata PRMetadata + diff string + files map[string]string // relative path -> content, written into the snapshot dir + headSHA string + err error +} + +func (f *fakePRSource) Metadata(context.Context, string, string, int) (PRMetadata, error) { + return f.metadata, f.err +} + +func (f *fakePRSource) Diff(context.Context, string, string, int) (string, error) { + return f.diff, f.err +} + +func (f *fakePRSource) Snapshot(_ context.Context, _, _ string, _ int, destDir string) (string, error) { + if f.err != nil { + return "", f.err + } + for path, content := range f.files { + if err := os.WriteFile(destDir+"/"+path, []byte(content), 0o644); err != nil { + return "", err + } + } + return f.headSHA, nil +} + +// concurrentAgentRunner counts simultaneous Run calls and dispatches by the +// request's Stage name -- pr-review names every child step after its own +// phase/dimension, so tests can script one behaviour per call site. +type concurrentAgentRunner struct { + mu sync.Mutex + current int + peak int + + byStage func(stage string) (agentexec.Result, error) +} + +func (r *concurrentAgentRunner) Run( + _ context.Context, _ string, req agentexec.Request, _ agentexec.ToolCallReporter, +) (agentexec.Result, error) { + r.mu.Lock() + r.current++ + if r.current > r.peak { + r.peak = r.current + } + r.mu.Unlock() + defer func() { + r.mu.Lock() + r.current-- + r.mu.Unlock() + }() + res, err := r.byStage(req.Stage) + res.Version, res.TaskID, res.Attempt, res.Stage = agentexec.ProtocolVersion, req.TaskID, req.Attempt, req.Stage + return res, err +} + +// threadSafeStore is a Store fake safe for the concurrent StepExecution +// writes phase 3, 4 and 9's bounded fan-out makes from goroutines -- +// recordingStore (changes_test.go) is not, and races under -race. +type threadSafeStore struct { + mu sync.Mutex + nextID int64 + starts []StepStart + finishes []StepFinish +} + +func (s *threadSafeStore) Update(context.Context, *Task) error { return nil } +func (s *threadSafeStore) Transition(context.Context, int64, string, string, string) error { + return nil +} + +func (s *threadSafeStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil } + +func (s *threadSafeStore) StartStep(_ context.Context, start StepStart) (int64, events.Event, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.nextID++ + s.starts = append(s.starts, start) + return s.nextID, events.Event{}, nil +} + +func (s *threadSafeStore) FinishStep(_ context.Context, finish StepFinish) (events.Event, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.finishes = append(s.finishes, finish) + return events.Event{}, nil +} + +func passedResult(summary string) agentexec.Result { + return agentexec.Result{Status: agentexec.StatusPassed, Summary: summary} +} + +func captureResult(tool string, payload any) agentexec.Result { + data, err := json.Marshal(payload) + if err != nil { + panic(err) + } + return agentexec.Result{ + Status: agentexec.StatusPassed, Summary: "ok", + Captures: map[string][]json.RawMessage{tool: {data}}, + } +} + +func baseTaskContext(t *testing.T) *TaskContext { + t.Helper() + return &TaskContext{ + Task: &Task{ID: 1, Attempt: 1, Owner: "acme", Repo: "widgets", PRNumber: 42, Workflow: "pr-review"}, + Cfg: config.Config{ + Models: map[string]string{"review": "provider/review", "classification": "provider/classification"}, + }, + Store: &threadSafeStore{}, + Log: slog.New(slog.DiscardHandler), + } +} + +func TestStagePRIntakeClassifiesDepthAndScoresAIGenerated(t *testing.T) { + tc := baseTaskContext(t) + tc.PRSource = &fakePRSource{ + metadata: PRMetadata{Title: "t", Body: "b"}, + diff: smallDiff, + } + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + return captureResult("score_ai_generated", map[string]any{"confidence": 0.9}), nil + }} + + if err := stagePRIntake().Run(context.Background(), tc); err != nil { + t.Fatalf("intake: %v", err) + } + if tc.prReview == nil { + t.Fatal("intake left no prReview state") + } + if tc.prReview.depth != prreview.DepthQuick { + t.Errorf("depth = %q, want quick for a %d-line diff", tc.prReview.depth, tc.prReview.stats.TotalAdditions+tc.prReview.stats.TotalDeletions) + } + if tc.prReview.aiGenerated != 0.9 { + t.Errorf("aiGenerated = %v, want 0.9 (from the captured score)", tc.prReview.aiGenerated) + } +} + +func TestStagePRIntakeFailsWithNoPRSource(t *testing.T) { + tc := baseTaskContext(t) + if err := stagePRIntake().Run(context.Background(), tc); err == nil { + t.Fatal("want an error with no PRSource configured") + } +} + +func TestStagePRAnatomyBuildsSnapshotBlastRadiusAndNarrative(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{ + diff: smallDiff, + files: prreview.ParseDiff(smallDiff), + } + tc.PRSource = &fakePRSource{ + headSHA: "deadbeef", + files: map[string]string{"main.go": "package main\n"}, + } + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + return passedResult("narrative here"), nil + }} + + if err := stagePRAnatomy().Run(context.Background(), tc); err != nil { + t.Fatalf("anatomy: %v", err) + } + if tc.prReview.headSHA != "deadbeef" { + t.Errorf("headSHA = %q, want deadbeef", tc.prReview.headSHA) + } + if tc.prReview.snapshotDir == "" { + t.Fatal("no snapshot directory recorded") + } + defer os.RemoveAll(tc.prReview.snapshotDir) + if _, err := os.Stat(tc.prReview.snapshotDir + "/main.go"); err != nil { + t.Errorf("snapshot missing main.go: %v", err) + } + if tc.prReview.narrative != "narrative here" { + t.Errorf("narrative = %q, want the agent's summary", tc.prReview.narrative) + } +} + +func TestStagePRLensesMergesCapsAndAddsHallucinationDimension(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{depth: prreview.DepthQuick, aiGenerated: 0.9, snapshotDir: t.TempDir()} + + var calls atomic.Int32 + tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) { + calls.Add(1) + // Priority 0.5, below the hallucination-check dimension's fixed + // priority of 1 (dimension.go), so the cap's cut is deterministic: + // the hallucination dimension always survives, one lens dimension + // always does not. + return captureResult("propose_dimensions", map[string]any{ + "dimensions": []map[string]any{ + {"name": stage, "prompt": "p", "target_files": []string{"a.go"}, "priority": 0.5}, + }, + }), nil + }} + + if err := stagePRLenses().Run(context.Background(), tc); err != nil { + t.Fatalf("lenses: %v", err) + } + if calls.Load() != 3 { + t.Errorf("agent calls = %d, want 3 (one per lens)", calls.Load()) + } + // quick depth caps at 3 dimensions total; three lenses each proposed one + // distinctly named dimension, plus the hallucination-check dimension a + // 0.9 AI-generated score adds as a fourth candidate -- capped to 3, not + // 3 lens dimensions plus the hallucination dimension on top. + if len(tc.prReview.dimensions) != prreview.MaxDimensionsFor(prreview.DepthQuick) { + t.Fatalf("dimensions = %d, want the quick cap of %d", len(tc.prReview.dimensions), prreview.MaxDimensionsFor(prreview.DepthQuick)) + } + foundHallucination := false + for _, d := range tc.prReview.dimensions { + if d.Name == "hallucination-check" { + foundHallucination = true + } + } + if !foundHallucination { + t.Error("hallucination-check dimension was cut even though its priority of 1 outranks every lens dimension's 0.5") + } +} + +func TestStagePRReviewBoundsConcurrencyAndDistinguishesTimeoutFromCleanEmpty(t *testing.T) { + tc := baseTaskContext(t) + dims := make([]prreview.Dimension, 10) + for i := range dims { + dims[i] = prreview.Dimension{Name: fmt.Sprintf("dim-%d", i), Prompt: "p", TargetFiles: []string{"a.go"}} + } + tc.prReview = &prReviewState{dimensions: dims, snapshotDir: t.TempDir()} + + runner := &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) { + if stage == "reviewer-dim-0" { + // Simulate a reviewer that exhausted its turn cap before its + // terminal tool call: no Go error, but not agentexec.StatusPassed. + return agentexec.Result{Status: agentexec.StatusParked, StopReason: "budget_exhausted"}, nil + } + return captureResult("report_findings", map[string]any{"findings": []any{}}), nil + }} + tc.Agent = runner + store := &threadSafeStore{} + tc.Store = store + tc.StepID = 1 // non-zero, so startChildStep/finishChildStep actually record + + if err := stagePRReview().Run(context.Background(), tc); err != nil { + t.Fatalf("review: %v", err) + } + if runner.peak > prReviewConcurrency { + t.Errorf("peak concurrent reviewer calls = %d, want at most %d", runner.peak, prReviewConcurrency) + } + if len(tc.prReview.findings) != 0 { + t.Errorf("findings = %d, want 0 (every reviewer reported none)", len(tc.prReview.findings)) + } + + store.mu.Lock() + defer store.mu.Unlock() + var timedOut, cleanEmpty int + for _, f := range store.finishes { + switch f.To { + case taskstate.StepFailed: + timedOut++ + case taskstate.StepSucceeded: + cleanEmpty++ + default: + t.Errorf("unexpected finish status %q", f.To) + } + } + if timedOut != 1 { + t.Errorf("StepFailed finishes = %d, want exactly 1 (the turn-cap-exhausted reviewer)", timedOut) + } + if cleanEmpty != 9 { + t.Errorf("clean StepSucceeded finishes = %d, want 9", cleanEmpty) + } +} + +func TestStagePRSynthesisScoresAndCapsFindings(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{ + aiGenerated: 0.9, + blastRadius: make([]string, 12), + findings: []prreview.Finding{ + {File: "a.go", LineStart: 1, Severity: prreview.SeverityCritical, Title: "t", Body: "b", Evidence: "e", Confidence: 0.9}, + }, + } + if err := stagePRSynthesis().Run(context.Background(), tc); err != nil { + t.Fatalf("synthesis: %v", err) + } + if len(tc.prReview.scored) != 1 { + t.Fatalf("scored = %d, want 1", len(tc.prReview.scored)) + } + // blast radius over 10 files and a likely machine-written PR both apply + // their multiplier (scoring.go), so the score exceeds the base severity + // weight x confidence (1.0 x 0.9 = 0.9). + if tc.prReview.scored[0].Score <= 0.9 { + t.Errorf("score = %v, want more than the unmultiplied 0.9 (blast radius + AI-generated multipliers should have applied)", tc.prReview.scored[0].Score) + } +} + +func TestStagePROutputPolishesPostsAndCompletes(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{ + headSHA: "deadbeef", + scored: []prreview.ScoredFinding{ + {File: "a.go", LineStart: 3, Title: "t1", Body: "original 1"}, + {File: "b.go", LineStart: 5, Title: "t2", Body: "original 2"}, + }, + } + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + return captureResult("polish", map[string]any{"body": "tightened"}), nil + }} + forge := &fakeForger{} + tc.Forge = forge + + if err := stagePROutput().Run(context.Background(), tc); err != nil { + t.Fatalf("output: %v", err) + } + for _, f := range tc.prReview.scored { + if f.Body != "tightened" { + t.Errorf("finding %q body = %q, want the polish call's tightened body", f.Title, f.Body) + } + } + if len(forge.comments) != 2 { + t.Fatalf("posted comments = %d, want 2", len(forge.comments)) + } + if forge.reviewedHeadSHA != "deadbeef" { + t.Errorf("posted headSHA = %q, want deadbeef", forge.reviewedHeadSHA) + } + if tc.Outcome.Status != StatusCompleted { + t.Errorf("outcome status = %q, want %q", tc.Outcome.Status, StatusCompleted) + } +} + +func TestStagePROutputKeepsOriginalBodyWhenPolishFails(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{scored: []prreview.ScoredFinding{{File: "a.go", LineStart: 1, Body: "original"}}} + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + return agentexec.Result{}, fmt.Errorf("model unavailable") + }} + tc.Forge = &fakeForger{} + + if err := stagePROutput().Run(context.Background(), tc); err != nil { + t.Fatalf("output: %v", err) + } + if tc.prReview.scored[0].Body != "original" { + t.Errorf("body = %q, want the original body kept on polish failure", tc.prReview.scored[0].Body) + } +} + +// fakeForger records CreateReviewComments calls; every other Forger method +// panics if called, since no pr-review stage should reach them. +type fakeForger struct { + comments []ReviewComment + reviewedHeadSHA string +} + +func (f *fakeForger) CloseIssue(context.Context, string, string, int, string) error { panic("unused") } + +func (f *fakeForger) CreatePR(context.Context, string, string, string, string, string, string) (int, error) { + panic("unused") +} + +func (f *fakeForger) LinkBranch(context.Context, string, string, int, string) error { panic("unused") } + +func (f *fakeForger) CreateReviewComments( + _ context.Context, _, _ string, _ int, reviewedHeadSHA string, comments []ReviewComment, +) error { + f.comments = comments + f.reviewedHeadSHA = reviewedHeadSHA + return nil +} + +func (f *fakeForger) Comment(context.Context, string, string, int, string) (int64, error) { + panic("unused") +} + +func (f *fakeForger) ReplyToReview(context.Context, string, string, int, int64, string) error { + panic("unused") +} + +// smallDiff is a one-line, one-file diff: under the 100-line quick-depth +// ceiling, so intake classifies it as quick. +const smallDiff = `diff --git a/main.go b/main.go +index e69de29..4b825dc 100644 +--- a/main.go ++++ b/main.go +@@ -0,0 +1 @@ ++package main +` diff --git a/internal/domain/workflow/workflow.go b/internal/domain/workflow/workflow.go index e308e2828..3c4046c2c 100644 --- a/internal/domain/workflow/workflow.go +++ b/internal/domain/workflow/workflow.go @@ -93,6 +93,17 @@ type TaskContext struct { // rather than silently skipping if it is enabled with no Reviewer // wired. Reviewer Reviewer + // PRSource fetches a pull request under review and its head snapshot for + // the pr-review workflow (see prreview_stages.go). Nil is only safe for a + // workflow that never runs pr-review's stages: they fail the run with a + // named error rather than silently skipping, the same rule Reviewer and + // Calls follow. + PRSource PRSource + // prReview is the pr-review workflow's cross-stage scratch state (see + // prreview_stages.go). It is unexported, following the pattern set by + // the feasibility workflow's decision field: pipeline-private state has + // no business being read or set outside its own stages. + prReview *prReviewState Bus *events.Bus // nil-safe via Emit Log *slog.Logger // CustomStages is retained for source compatibility only. Repository Go