feat(workflow): merge gate, budget shares and skipped-phase reporting for pr-review
Phase 8 (merge gate) runs one classification call per finding, deciding blocking vs advisory per the PRD's narrow criteria; a failed or malformed call leaves the finding advisory. Added between synthesis and output. A run-level token/wall-clock budget, split into cumulative per-phase shares (prreview.Budget.PhaseExhausted), gates the expensive agent-call work in lenses, review, verification, coverage-consistency, merge-gate, and the anatomy narrative / output polish sub-calls -- deterministic bookkeeping (snapshot, blast radius, clustering, scoring) always runs regardless. A skipped phase is recorded and named in the posted review's outcome, alongside the REQUEST_CHANGES/COMMENT event ReviewEventFor already decided. Known gaps left for a follow-up: workflow.Forger.CreateReviewComments always posts a COMMENT-state review (GitHub's implementation submits per-comment calls, not a review object, so REQUEST_CHANGES needs a Forger/forge change), and there is no PR-body fallback list for whole-file findings (docs/prds/inline-review.md).
This commit is contained in:
6 files changed
+466
-26
No files matched your search
@@ -0,0 +1,36 @@
|
||||
package prreview
|
||||
|
||||
import "time"
|
||||
|
||||
// Budget bounds a pr-review run's total cost (tokens, the only spend the
|
||||
// agent runtime accounts for) and wall-clock, split evenly across every
|
||||
// budget-tracked phase (docs/prds/pr-review-agent.md, Budget: "A run has a
|
||||
// cost cap and a wall-clock cap, split into per-phase shares"). Zero
|
||||
// disables that dimension.
|
||||
type Budget struct {
|
||||
MaxTokens int
|
||||
WallClock time.Duration
|
||||
}
|
||||
|
||||
// PhaseExhausted reports whether phase (1-based, counting the phase about to
|
||||
// start) has already spent its share, given how much of the run's total
|
||||
// budget every phase up to and including it is allotted out of phases total.
|
||||
//
|
||||
// The check is against the cumulative allotment through phase, not phase's
|
||||
// own isolated slice: an earlier phase's underspend is not wasted, but its
|
||||
// overspend still eats into a later phase's share. That is what "a phase
|
||||
// whose share is spent is skipped" means for a phase that has not started
|
||||
// yet and so has spent nothing of its own -- the run's spend so far is
|
||||
// compared against how much the whole run should have spent by this point.
|
||||
func (b Budget) PhaseExhausted(phase, phases, tokensSpent int, elapsed time.Duration) bool {
|
||||
if phase <= 0 || phases <= 0 {
|
||||
return false
|
||||
}
|
||||
if b.MaxTokens > 0 && tokensSpent >= b.MaxTokens*phase/phases {
|
||||
return true
|
||||
}
|
||||
if b.WallClock > 0 && elapsed >= b.WallClock*time.Duration(phase)/time.Duration(phases) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package prreview
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBudgetPhaseExhausted(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
budget Budget
|
||||
phase int
|
||||
phases int
|
||||
tokensSpent int
|
||||
elapsed time.Duration
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "zero budget never exhausts",
|
||||
budget: Budget{},
|
||||
phase: 1, phases: 6,
|
||||
tokensSpent: 1_000_000, elapsed: time.Hour,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "fresh phase under its cumulative token allotment",
|
||||
budget: Budget{MaxTokens: 600},
|
||||
phase: 1, phases: 6, // allotted 100 tokens by phase 1
|
||||
tokensSpent: 50,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "tokens spent already meets the cumulative allotment",
|
||||
budget: Budget{MaxTokens: 600},
|
||||
phase: 1, phases: 6, // allotted 100 tokens by phase 1
|
||||
tokensSpent: 100,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "later phase has a larger cumulative allotment",
|
||||
budget: Budget{MaxTokens: 600},
|
||||
phase: 3, phases: 6, // allotted 300 tokens by phase 3
|
||||
tokensSpent: 250,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "wall clock exhausted independently of tokens",
|
||||
budget: Budget{WallClock: 60 * time.Minute},
|
||||
phase: 2, phases: 6, // allotted 20 minutes by phase 2
|
||||
elapsed: 25 * time.Minute,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "wall clock not yet exhausted",
|
||||
budget: Budget{WallClock: 60 * time.Minute},
|
||||
phase: 2, phases: 6,
|
||||
elapsed: 10 * time.Minute,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "phase zero is never exhausted (guard)",
|
||||
budget: Budget{MaxTokens: 1},
|
||||
phase: 0,
|
||||
phases: 6,
|
||||
tokensSpent: 1_000_000,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "phases zero is never exhausted (guard)",
|
||||
budget: Budget{MaxTokens: 1},
|
||||
phase: 1,
|
||||
phases: 0,
|
||||
tokensSpent: 1_000_000,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := c.budget.PhaseExhausted(c.phase, c.phases, c.tokensSpent, c.elapsed)
|
||||
if got != c.want {
|
||||
t.Errorf("PhaseExhausted(%d, %d, %d, %v) = %v, want %v",
|
||||
c.phase, c.phases, c.tokensSpent, c.elapsed, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,11 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
@@ -75,6 +78,60 @@ type prReviewState struct {
|
||||
// parallel (docs/prds/pr-review-agent.md, phase 6). Every other phase
|
||||
// touches findings from a single goroutine and does not need it.
|
||||
findingsMu sync.Mutex
|
||||
|
||||
// runStart is when phase 1 started, the origin every budget-share check
|
||||
// measures elapsed wall-clock against.
|
||||
runStart time.Time
|
||||
// tokensSpent is the run's cumulative token spend so far, across every
|
||||
// agent call any phase has made. It is an atomic because phases 3, 4 and
|
||||
// 9 add to it from concurrent goroutines.
|
||||
tokensSpent atomic.Int64
|
||||
// skippedPhases names every phase a budget check skipped, in the order
|
||||
// they were skipped, so phase 9's output can report them (docs/prds/
|
||||
// pr-review-agent.md, Budget: "The posted review names every skipped
|
||||
// phase, so an exhausted run never reads as a clean one"). Every phase
|
||||
// runs sequentially in the workflow engine's stage loop, so appends here
|
||||
// need no lock.
|
||||
skippedPhases []string
|
||||
}
|
||||
|
||||
// prReviewTotalBudget bounds the whole pr-review run's cost (tokens, the
|
||||
// only spend the agent runtime accounts for) and wall-clock, split evenly
|
||||
// across prReviewBudgetPhases in pipeline order (docs/prds/pr-review-agent.md,
|
||||
// Budget section). Sized generously: exhausting it is meant to catch a run
|
||||
// that is genuinely stuck or unexpectedly expensive, not a typical one.
|
||||
var prReviewTotalBudget = prreview.Budget{MaxTokens: 2_000_000, WallClock: 30 * time.Minute}
|
||||
|
||||
// prReviewBudgetPhases are the budget-tracked phases, in pipeline order.
|
||||
// Intake is not tracked: it must always run, since the depth, the AI-score
|
||||
// and the budget's own runStart all come from it, and every later phase and
|
||||
// this list itself depend on its output existing.
|
||||
var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "verification", "coverage-consistency", "merge-gate", "output"}
|
||||
|
||||
// budgetExhausted reports whether phase has already spent its cumulative
|
||||
// share of the run's total budget, checked once before that phase's
|
||||
// expensive (agent-call) work starts. A phase absent from
|
||||
// prReviewBudgetPhases is a bug in that list, not a runtime condition, and
|
||||
// is never treated as exhausted.
|
||||
func budgetExhausted(tc *TaskContext, phase string) bool {
|
||||
idx := slices.Index(prReviewBudgetPhases, phase)
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
// A zero runStart means the run's clock was never started (a stage
|
||||
// invoked directly against a hand-built prReviewState, as the stage unit
|
||||
// tests do) rather than a run that has been going since the epoch.
|
||||
var elapsed time.Duration
|
||||
if !tc.prReview.runStart.IsZero() {
|
||||
elapsed = time.Since(tc.prReview.runStart)
|
||||
}
|
||||
return prReviewTotalBudget.PhaseExhausted(idx+1, len(prReviewBudgetPhases),
|
||||
int(tc.prReview.tokensSpent.Load()), elapsed)
|
||||
}
|
||||
|
||||
// skipPhase records phase as budget-skipped.
|
||||
func skipPhase(tc *TaskContext, phase string) {
|
||||
tc.prReview.skippedPhases = append(tc.prReview.skippedPhases, phase)
|
||||
}
|
||||
|
||||
// prReviewConcurrency bounds phases 3 (lenses), 4 (reviewers) and 9 (polish),
|
||||
@@ -84,8 +141,9 @@ const prReviewConcurrency = 8
|
||||
|
||||
// PRReview is the pull request reviewer: a fixed pipeline, not a single
|
||||
// agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2,
|
||||
// 3, 4, 7 and 9; archie-core-afbk.4 adds 5 and 6 (verification,
|
||||
// coverage/consistency); phase 8 (the merge gate) is archie-core-afbk.5.
|
||||
// 3, 4, 7 and 9; archie-core-afbk.4 added 5 and 6 (verification,
|
||||
// coverage/consistency); archie-core-afbk.5 adds phase 8 (the merge gate)
|
||||
// and the budget shares/skipped-phase reporting that run throughout.
|
||||
func PRReview() Workflow {
|
||||
return Workflow{
|
||||
Name: "pr-review",
|
||||
@@ -97,6 +155,7 @@ func PRReview() Workflow {
|
||||
stagePRVerification(),
|
||||
stagePRCoverageConsistency(),
|
||||
stagePRSynthesis(),
|
||||
stagePRMergeGate(),
|
||||
stagePROutput(),
|
||||
},
|
||||
}
|
||||
@@ -122,15 +181,19 @@ func stagePRIntake() Stage {
|
||||
stats := prreview.SummarizeFiles(files)
|
||||
depth := prreview.ResolveDepth(stats.TotalAdditions+stats.TotalDeletions, "")
|
||||
|
||||
// tc.prReview must exist before scoreAIGenerated runs: every
|
||||
// pr-review agent call, including this one, records its token spend
|
||||
// against tc.prReview.tokensSpent.
|
||||
tc.prReview = &prReviewState{
|
||||
metadata: meta, diff: diff, files: files, stats: stats,
|
||||
depth: depth, runStart: time.Now(),
|
||||
}
|
||||
|
||||
aiGenerated, err := scoreAIGenerated(ctx, tc, meta)
|
||||
if err != nil {
|
||||
return fmt.Errorf("score likely machine-written PR: %w", err)
|
||||
}
|
||||
|
||||
tc.prReview = &prReviewState{
|
||||
metadata: meta, diff: diff, files: files, stats: stats,
|
||||
depth: depth, aiGenerated: aiGenerated,
|
||||
}
|
||||
tc.prReview.aiGenerated = aiGenerated
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
@@ -227,6 +290,10 @@ func stagePRAnatomy() Stage {
|
||||
"Call finish with status \"passed\" and your narrative as the summary.",
|
||||
tc.prReview.metadata.Title, tc.prReview.metadata.Body, clip(tc.prReview.diff, 60000),
|
||||
)
|
||||
if budgetExhausted(tc, "anatomy") {
|
||||
skipPhase(tc, "anatomy")
|
||||
return nil
|
||||
}
|
||||
res, err := runPRReviewAgent(ctx, tc, tc.prReview.snapshotDir, "anatomy", "review", mission, 15, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -255,6 +322,10 @@ var prReviewLenses = []struct {
|
||||
// -- so it is added before MergeDimensions cuts the tail, never after.
|
||||
func stagePRLenses() Stage {
|
||||
return Stage{Name: "lenses", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if budgetExhausted(tc, "lenses") {
|
||||
skipPhase(tc, "lenses")
|
||||
return nil
|
||||
}
|
||||
outputs := make([][]prreview.Dimension, len(prReviewLenses))
|
||||
errs := make([]error, len(prReviewLenses))
|
||||
forEachBounded(len(prReviewLenses), len(prReviewLenses), func(i int) {
|
||||
@@ -347,6 +418,10 @@ func runLens(ctx context.Context, tc *TaskContext, name, angle string) ([]prrevi
|
||||
// (docs/prds/pr-review-agent.md, phase 4).
|
||||
func stagePRReview() Stage {
|
||||
return Stage{Name: "review", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if budgetExhausted(tc, "review") {
|
||||
skipPhase(tc, "review")
|
||||
return nil
|
||||
}
|
||||
dimensions := tc.prReview.dimensions
|
||||
findings := make([][]prreview.Finding, len(dimensions))
|
||||
forEachBounded(prReviewConcurrency, len(dimensions), func(i int) {
|
||||
@@ -501,6 +576,7 @@ func runReviewerAgent(
|
||||
return agentexec.Result{}, err
|
||||
}
|
||||
res, runErr := tc.Agent.Run(ctx, tc.prReview.snapshotDir, req, tc.toolCallReporter(name))
|
||||
tc.prReview.tokensSpent.Add(int64(res.TokensUsed))
|
||||
to, detail := taskstate.StepSucceeded, res.Summary
|
||||
switch {
|
||||
case runErr != nil:
|
||||
@@ -511,15 +587,15 @@ func runReviewerAgent(
|
||||
if ferr := tc.finishChildStep(ctx, stepID, to, detail, int64(res.TokensUsed)); ferr != nil {
|
||||
return res, ferr
|
||||
}
|
||||
tc.prReview.tokensSpent.Add(int64(res.TokensUsed))
|
||||
return res, runErr
|
||||
}
|
||||
|
||||
// stagePRSynthesis is pipeline phase 7, code only: score, drop findings
|
||||
// under their severity's confidence floor, merge duplicates, rank, and cap
|
||||
// at the inline comment limit. Phase 8 (the merge gate) does not run yet, so
|
||||
// every surviving finding's Blocking flag is whatever phase 5/6 left it as
|
||||
// (false, since neither sets it -- that is archie-core-afbk.5's decision to
|
||||
// make).
|
||||
// at the inline comment limit. Phase 8 (the merge gate) runs after this
|
||||
// stage, so every surviving finding's Blocking flag is still whatever phase
|
||||
// 5/6 left it as (false, since neither sets it) until that stage decides.
|
||||
func stagePRSynthesis() Stage {
|
||||
return Stage{Name: "synthesis", Run: func(_ context.Context, tc *TaskContext) error {
|
||||
scored := prreview.Score(tc.prReview.findings, prreview.ScoreInputs{
|
||||
@@ -534,24 +610,29 @@ func stagePRSynthesis() Stage {
|
||||
// stagePROutput is pipeline phase 9: one polish call per comment, at most
|
||||
// prReviewConcurrency at a time, keeping the original wording on failure,
|
||||
// then posting the surviving inline-anchored findings as one forge review.
|
||||
// The merge gate (phase 8) does not run yet, so there is no blocking
|
||||
// classification to decide REQUEST_CHANGES; every posted review is a
|
||||
// comment until archie-core-afbk.5 lands that decision.
|
||||
// The polish pass is what a budget-exhausted run skips; posting itself
|
||||
// always runs, since it is the pipeline's only externally visible act and
|
||||
// must report what happened even when every other phase was skipped.
|
||||
func stagePROutput() Stage {
|
||||
return Stage{Name: "output", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
polished := make([]prreview.ScoredFinding, len(tc.prReview.scored))
|
||||
forEachBounded(prReviewConcurrency, len(tc.prReview.scored), func(i int) {
|
||||
polished[i] = polishFinding(ctx, tc, tc.prReview.scored[i])
|
||||
})
|
||||
tc.prReview.scored = polished
|
||||
if budgetExhausted(tc, "output") {
|
||||
skipPhase(tc, "output")
|
||||
} else {
|
||||
polished := make([]prreview.ScoredFinding, len(tc.prReview.scored))
|
||||
forEachBounded(prReviewConcurrency, len(tc.prReview.scored), func(i int) {
|
||||
polished[i] = polishFinding(ctx, tc, tc.prReview.scored[i])
|
||||
})
|
||||
tc.prReview.scored = polished
|
||||
}
|
||||
|
||||
if err := postPRReview(ctx, tc); err != nil {
|
||||
return fmt.Errorf("post review: %w", err)
|
||||
}
|
||||
tc.Outcome = Outcome{
|
||||
Status: StatusCompleted,
|
||||
Detail: fmt.Sprintf("posted %d finding(s)", len(polished)),
|
||||
detail := fmt.Sprintf("posted %d finding(s) as %s", len(tc.prReview.scored), prreview.ReviewEventFor(tc.prReview.scored))
|
||||
if len(tc.prReview.skippedPhases) > 0 {
|
||||
detail += fmt.Sprintf("; skipped: %s", strings.Join(tc.prReview.skippedPhases, ", "))
|
||||
}
|
||||
tc.Outcome = Outcome{Status: StatusCompleted, Detail: detail}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
@@ -593,9 +674,14 @@ func polishFinding(ctx context.Context, tc *TaskContext, f prreview.ScoredFindin
|
||||
}
|
||||
|
||||
// postPRReview posts every line-anchored finding as one forge review. A
|
||||
// finding with no line (LineStart <= 0) has nothing to post yet: the PR-body
|
||||
// fallback list is the merge gate stage's concern (archie-core-afbk.5), not
|
||||
// this bead's.
|
||||
// finding with no line (LineStart <= 0) is dropped rather than posted
|
||||
// anywhere -- docs/prds/inline-review.md's PR-body fallback list for
|
||||
// whole-file findings, and REQUEST_CHANGES vs COMMENT event submission
|
||||
// (workflow.Forger.CreateReviewComments always posts a COMMENT-state
|
||||
// review; GitHub's implementation does not submit a review object at all,
|
||||
// only per-comment calls, so REQUEST_CHANGES needs a Forger/forge change,
|
||||
// not just a caller change here) are both real gaps this bead does not
|
||||
// close; see the follow-up bead this bead's commit files.
|
||||
func postPRReview(ctx context.Context, tc *TaskContext) error {
|
||||
comments := make([]ReviewComment, 0, len(tc.prReview.scored))
|
||||
for _, f := range tc.prReview.scored {
|
||||
@@ -657,9 +743,11 @@ func runPRReviewAgentRecorded(
|
||||
Budget: agentexec.Budget{MaxSteps: maxSteps, WallClock: tc.Cfg.Budgets.WallClock.Std()},
|
||||
CaptureTools: captureTools,
|
||||
}
|
||||
return tc.RunAgentChild(ctx, name, func() (agentexec.Result, error) {
|
||||
res, err := tc.RunAgentChild(ctx, name, func() (agentexec.Result, error) {
|
||||
return tc.Agent.Run(ctx, workspace, req, tc.toolCallReporter(name))
|
||||
})
|
||||
tc.prReview.tokensSpent.Add(int64(res.TokensUsed))
|
||||
return res, err
|
||||
}
|
||||
|
||||
// forEachBounded runs fn(0), fn(1), ..., fn(n-1), at most limit at a time,
|
||||
|
||||
@@ -6,9 +6,12 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
@@ -366,6 +369,100 @@ func TestStagePROutputKeepsOriginalBodyWhenPolishFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetExhaustedSkipsLensesAndRecordsIt(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
depth: prreview.DepthQuick, snapshotDir: t.TempDir(), runStart: time.Now(),
|
||||
}
|
||||
// Spend past every phase's cumulative token allotment up front, so
|
||||
// lenses (the first budget-tracked phase) is already exhausted before
|
||||
// its own work would start.
|
||||
tc.prReview.tokensSpent.Store(int64(prReviewTotalBudget.MaxTokens))
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no lens call should run once the budget is already spent")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRLenses().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("lenses: %v", err)
|
||||
}
|
||||
if len(tc.prReview.dimensions) != 0 {
|
||||
t.Errorf("dimensions = %d, want 0 (lenses skipped, not run)", len(tc.prReview.dimensions))
|
||||
}
|
||||
if len(tc.prReview.skippedPhases) != 1 || tc.prReview.skippedPhases[0] != "lenses" {
|
||||
t.Errorf("skippedPhases = %v, want [\"lenses\"]", tc.prReview.skippedPhases)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBudgetNotExhaustedRunsLensesNormally(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{depth: prreview.DepthQuick, snapshotDir: t.TempDir(), runStart: time.Now()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("propose_dimensions", map[string]any{
|
||||
"dimensions": []map[string]any{{"name": "d", "prompt": "p", "target_files": []string{"a.go"}, "priority": 1.0}},
|
||||
}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRLenses().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("lenses: %v", err)
|
||||
}
|
||||
if len(tc.prReview.skippedPhases) != 0 {
|
||||
t.Errorf("skippedPhases = %v, want none: the budget was not spent", tc.prReview.skippedPhases)
|
||||
}
|
||||
if len(tc.prReview.dimensions) == 0 {
|
||||
t.Error("dimensions = 0, want at least the lens's own proposal: lenses should have run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROutputReportsSkippedPhasesAndReviewEvent(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
headSHA: "deadbeef",
|
||||
skippedPhases: []string{"verification", "coverage-consistency"},
|
||||
scored: []prreview.ScoredFinding{
|
||||
{File: "a.go", LineStart: 1, Title: "t", Body: "b", Blocking: true},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return captureResult("polish", map[string]any{"body": "tightened"}), nil
|
||||
}}
|
||||
tc.Forge = &fakeForger{}
|
||||
|
||||
if err := stagePROutput().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("output: %v", err)
|
||||
}
|
||||
if !strings.Contains(tc.Outcome.Detail, "REQUEST_CHANGES") {
|
||||
t.Errorf("outcome detail = %q, want it to name REQUEST_CHANGES (a blocking finding survived)", tc.Outcome.Detail)
|
||||
}
|
||||
if !strings.Contains(tc.Outcome.Detail, "verification") || !strings.Contains(tc.Outcome.Detail, "coverage-consistency") {
|
||||
t.Errorf("outcome detail = %q, want it to name every skipped phase", tc.Outcome.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROutputSkipsPolishWhenBudgetExhausted(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
runStart: time.Now(),
|
||||
scored: []prreview.ScoredFinding{{File: "a.go", LineStart: 1, Title: "t", Body: "original"}},
|
||||
}
|
||||
tc.prReview.tokensSpent.Store(int64(prReviewTotalBudget.MaxTokens))
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no polish call should run once the budget is already spent")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
tc.Forge = &fakeForger{}
|
||||
|
||||
if err := stagePROutput().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("output: %v", err)
|
||||
}
|
||||
if tc.prReview.scored[0].Body != "original" {
|
||||
t.Errorf("body = %q, want the original (unpolished) body", tc.prReview.scored[0].Body)
|
||||
}
|
||||
if !slices.Contains(tc.prReview.skippedPhases, "output") {
|
||||
t.Errorf("skippedPhases = %v, want it to include \"output\"", tc.prReview.skippedPhases)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeForger records CreateReviewComments calls; every other Forger method
|
||||
// panics if called, since no pr-review stage should reach them.
|
||||
type fakeForger struct {
|
||||
|
||||
@@ -486,3 +486,68 @@ func runConsistencyVerification(ctx context.Context, tc *TaskContext) error {
|
||||
tc.prReview.findingsMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// mergeGateSchema is the merge gate's classification-call capture-tool
|
||||
// schema.
|
||||
var mergeGateSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"blocking": {"type": "boolean", "description": "true only for a broken build, a security hole, data loss, a contract break or a regression."}
|
||||
},
|
||||
"required": ["blocking"]
|
||||
}`)
|
||||
|
||||
// stagePRMergeGate is pipeline phase 8: one classification call per
|
||||
// surviving finding decides blocking or advisory (docs/prds/pr-review-
|
||||
// agent.md, phase 8). Blocking is narrow, and a failed or malformed call
|
||||
// leaves a finding advisory -- the same fallback every other pr-review
|
||||
// verdict call in this pipeline uses when its agent call could not run.
|
||||
func stagePRMergeGate() Stage {
|
||||
return Stage{Name: "merge-gate", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if len(tc.prReview.scored) == 0 {
|
||||
return nil
|
||||
}
|
||||
if budgetExhausted(tc, "merge-gate") {
|
||||
skipPhase(tc, "merge-gate")
|
||||
return nil
|
||||
}
|
||||
blocking := make([]bool, len(tc.prReview.scored))
|
||||
forEachBounded(prReviewConcurrency, len(tc.prReview.scored), func(i int) {
|
||||
blocking[i] = runMergeGateCall(ctx, tc, tc.prReview.scored[i])
|
||||
})
|
||||
for i := range tc.prReview.scored {
|
||||
tc.prReview.scored[i].Blocking = blocking[i]
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// runMergeGateCall runs one phase-8 classification call over one finding.
|
||||
func runMergeGateCall(ctx context.Context, tc *TaskContext, f prreview.ScoredFinding) bool {
|
||||
mission := fmt.Sprintf(
|
||||
"Decide whether this finding must block the change from merging. Blocking is "+
|
||||
"narrow: only a broken build, a security hole, data loss, a contract break, or "+
|
||||
"a regression qualifies. Everything else -- including a real but non-blocking "+
|
||||
"defect -- is advisory.\n\n[%s] %s (%s:%d)\n%s\n\n"+
|
||||
"Call classify_blocking exactly once, then call finish with status \"passed\".",
|
||||
f.Severity, f.Title, f.File, f.LineStart, f.Body,
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "merge-gate", "classification", mission, 6, []agentexec.CaptureTool{{
|
||||
Name: "classify_blocking", Description: "Record the blocking verdict. Call exactly once, before finish.",
|
||||
Parameters: mergeGateSchema, RequiredFields: []string{"blocking"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return false //nolint:nilerr // a merge-gate call that could not run leaves the finding advisory, per the PRD
|
||||
}
|
||||
calls := res.Captures["classify_blocking"]
|
||||
if len(calls) != 1 {
|
||||
return false
|
||||
}
|
||||
var captured struct {
|
||||
Blocking bool `json:"blocking"`
|
||||
}
|
||||
if json.Unmarshal(calls[0], &captured) != nil {
|
||||
return false
|
||||
}
|
||||
return captured.Blocking
|
||||
}
|
||||
@@ -2,8 +2,10 @@ package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
@@ -193,3 +195,69 @@ func TestCoverageGapCapsAtTwoRounds(t *testing.T) {
|
||||
t.Errorf("gap reviewer calls = %d, want exactly 2 (the round cap)", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRMergeGateSetsBlockingIndependentlyPerFinding(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
scored: []prreview.ScoredFinding{
|
||||
{File: "a.go", LineStart: 1, Title: "sql injection"},
|
||||
{File: "b.go", LineStart: 2, Title: "unused variable"},
|
||||
},
|
||||
}
|
||||
// One call per finding: alternate the verdict by call order, so a bug
|
||||
// that broadcasts one verdict to every finding (rather than one call
|
||||
// each) would leave both findings the same instead of split.
|
||||
var calls atomic.Int32
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
n := calls.Add(1)
|
||||
return captureResult("classify_blocking", map[string]any{"blocking": n == 1}), nil
|
||||
}}
|
||||
|
||||
if err := stagePRMergeGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("merge-gate: %v", err)
|
||||
}
|
||||
if calls.Load() != 2 {
|
||||
t.Fatalf("classification calls = %d, want 2 (one per finding)", calls.Load())
|
||||
}
|
||||
blockingCount := 0
|
||||
for _, f := range tc.prReview.scored {
|
||||
if f.Blocking {
|
||||
blockingCount++
|
||||
}
|
||||
}
|
||||
if blockingCount != 1 {
|
||||
t.Errorf("blocking findings = %d, want exactly 1 (one call returned true, the other false)", blockingCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRMergeGateFailedCallLeavesFindingAdvisory(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
scored: []prreview.ScoredFinding{{File: "a.go", LineStart: 1, Title: "t", Blocking: true}},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return agentexec.Result{}, errors.New("model unavailable")
|
||||
}}
|
||||
|
||||
if err := stagePRMergeGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("merge-gate: %v", err)
|
||||
}
|
||||
if tc.prReview.scored[0].Blocking {
|
||||
t.Error("Blocking = true, want false: a failed merge-gate call must leave the finding advisory, per the PRD")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRMergeGateSkipsEntirelyWhenNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call should run when there are no findings to classify")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRMergeGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("merge-gate: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user