feat(workflow): pr-review precision dial and operator-approval gate

Add review.precision_gate and review.approve_before_post to config,
threaded through TaskConfig/ForTask/ToConfig so the pr-review workflow
reads them under archie-agent too, not just in-process.

stagePRPrecisionGate runs between review and verification, filtering
findings down to concrete, evidenced defects when the dial is on;
stagePROperatorApproval ends the run in waiting_human right after
synthesis, before the merge gate, when approval is required and
findings exist. Both are off by default and fail open on a broken
agent call, matching the pipeline's existing recall-first fallbacks.

Filed follow-ups: afbk.11 for the actual operator-response resume
mechanics (post/reject/re-review with instructions, the 2-round cap --
genuinely needs either a small Task schema addition or a documented
Notes-encoding decision, a call for whoever wires afbk.7's triggers)
and afbk.12 for exposing the two flags in the dashboard Config tab.
This commit is contained in:
sam committed 2026-09-28 12:20:10 +10:00
1 parent 5368d41231
commit cd443673f1
8 files changed
+291 -7

No files matched your search

+1
View File
@@ -117,6 +117,7 @@ var reloadableFields = map[string]bool{
"Models": true,
"ModelLimits": true,
"Notify": true,
"Review": true,
// BotUser/BotEmail used to be poll-path only; ForTask now carries both
// into the task snapshot as well, because the sandboxed worker builds
// its own worktree.Manager and signs its own commits (agentworker
+21
View File
@@ -210,6 +210,23 @@ type Forge struct {
WebhookAddr string `toml:"webhook_addr" yaml:"webhook_addr"`
}
// Review configures the pr-review workflow (docs/prds/pr-review-agent.md).
// Both fields are off by default: the pipeline is recall-first, and posts
// automatically, unless an operator opts into either dial.
type Review struct {
// PrecisionGate runs a post-worthiness pass after phase 4 (review) and
// before phase 5 (verification): it keeps every concrete, evidenced
// defect, drops nitpicks, style and unverifiable claims, and keeps a
// finding when unsure. It trades recall for precision and cuts
// verification cost by filtering early.
PrecisionGate bool `toml:"precision_gate" yaml:"precision_gate"`
// ApproveBeforePost ends the pipeline in waiting_human right after
// synthesis, before the merge gate runs, so an operator can choose which
// findings to post, reject the review, or ask for a re-review with
// instructions. A review with no findings never waits.
ApproveBeforePost bool `toml:"approve_before_post" yaml:"approve_before_post"`
}
// Dispatch configures how archied discovers work and reflects task state
// onto the forge via labels and reactions.
type Dispatch struct {
@@ -486,6 +503,7 @@ type Config struct {
Forge Forge `toml:"forge" yaml:"forge"`
Dispatch Dispatch `toml:"dispatch" yaml:"dispatch"`
Review Review `toml:"review" yaml:"review"`
// Models maps a role ("planner", "builder", "triage") to a runtime
// model ref ("provider/model").
@@ -604,6 +622,7 @@ type TaskConfig struct {
Notify Notify `json:"notify"`
Forge TaskForge `json:"forge"`
ToolPolicy ToolPolicy `json:"tool_policy"`
Review Review `json:"review"`
}
// DiffCapOf returns a DiffCapLines value for n. It exists because the field is
@@ -644,6 +663,7 @@ func (c Config) ForTask() TaskConfig {
Notify: c.Notify,
Forge: TaskForge{Host: c.Forge.Host},
ToolPolicy: c.Tools.Policy,
Review: c.Review,
}
}
@@ -663,6 +683,7 @@ func (tc TaskConfig) ToConfig() Config {
Notify: tc.Notify,
Forge: Forge{Host: tc.Forge.Host},
Tools: ToolsConfig{Policy: tc.ToolPolicy},
Review: tc.Review,
}
}
+5 -3
View File
@@ -157,8 +157,9 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
Providers: map[string]Provider{
"secret": {APIKeyEnv: "TOP_SECRET_PROVIDER_TOKEN"},
},
NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"},
Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}},
NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"},
Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}},
Review: Review{PrecisionGate: true, ApproveBeforePost: true},
}
want := TaskConfig{
@@ -171,6 +172,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
Notify: cfg.Notify,
Forge: TaskForge{Host: cfg.Forge.Host},
ToolPolicy: cfg.Tools.Policy,
Review: cfg.Review,
}
got := cfg.ForTask()
if !reflect.DeepEqual(got, want) {
@@ -197,7 +199,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
if err != nil {
t.Fatal(err)
}
assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy")
assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy", "review")
var forgePayload map[string]json.RawMessage
var payload map[string]json.RawMessage
+1 -1
View File
@@ -2098,7 +2098,7 @@ func TestDispatchCapturesTheAttemptsEffectiveConfig(t *testing.T) {
// repositories, identities or lock state.
wantKeys := []string{
"bot_email", "bot_user", "budgets", "diff_cap_lines", "dispatch",
"forge", "model_limits", "models", "notify", "tool_policy",
"forge", "model_limits", "models", "notify", "review", "tool_policy",
}
gotKeys := make([]string, 0, len(doc))
for key := range doc {
@@ -42,6 +42,7 @@ var outcomeSites = []outcomeSite{
{"feasibility.go", "StatusClosedWontDo", "feasibility decided this work should not be done"},
{"feasibility.go", "StatusWaitingHuman", "the PRD is delivered and the run waits for a go/no-go"},
{"prreview_stages.go", "StatusCompleted", "the pr-review pipeline posted its findings (or found none to post)"},
{"prreview_stages_verify.go", "StatusWaitingHuman", "review.approve_before_post is set and synthesis produced findings: the run waits for an operator decision before the merge gate"},
{"remediate.go", "StatusParked", "the remediation round cap is spent"},
{"remediate.go", "StatusPROpen", "a remediation round pushed the task back to its open PR"},
{"review.go", "StatusParked", "review is enabled but no reviewer is wired"},
+8 -3
View File
@@ -106,7 +106,7 @@ var prReviewTotalBudget = prreview.Budget{MaxTokens: 2_000_000, WallClock: 30 *
// Intake is not tracked: it must always run, since the depth, the AI-score
// and the budget's own runStart all come from it, and every later phase and
// this list itself depend on its output existing.
var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "verification", "coverage-consistency", "merge-gate", "output"}
var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "precision-gate", "verification", "coverage-consistency", "merge-gate", "output"}
// budgetExhausted reports whether phase has already spent its cumulative
// share of the run's total budget, checked once before that phase's
@@ -142,8 +142,11 @@ const prReviewConcurrency = 8
// PRReview is the pull request reviewer: a fixed pipeline, not a single
// agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2,
// 3, 4, 7 and 9; archie-core-afbk.4 added 5 and 6 (verification,
// coverage/consistency); archie-core-afbk.5 adds phase 8 (the merge gate)
// and the budget shares/skipped-phase reporting that run throughout.
// coverage/consistency); archie-core-afbk.5 added phase 8 (the merge gate)
// and the budget shares/skipped-phase reporting that run throughout;
// archie-core-afbk.8 adds the precision dial (between review and
// verification) and the operator-approval gate (between synthesis and the
// merge gate), both off by default and each gated by its own config flag.
func PRReview() Workflow {
return Workflow{
Name: "pr-review",
@@ -152,9 +155,11 @@ func PRReview() Workflow {
stagePRAnatomy(),
stagePRLenses(),
stagePRReview(),
stagePRPrecisionGate(),
stagePRVerification(),
stagePRCoverageConsistency(),
stagePRSynthesis(),
stagePROperatorApproval(),
stagePRMergeGate(),
stagePROutput(),
},
@@ -0,0 +1,133 @@
package workflow
import (
"context"
"testing"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
)
func TestStagePRPrecisionGateSkippedWhenConfigOff(t *testing.T) {
tc := baseTaskContext(t)
tc.prReview = &prReviewState{
snapshotDir: t.TempDir(),
findings: []prreview.Finding{{Title: "f0", Severity: prreview.SeverityNitpick}},
}
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
t.Fatal("no agent call expected: review.precision_gate is off")
return agentexec.Result{}, nil
}}
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
t.Fatalf("precision-gate: %v", err)
}
if len(tc.prReview.findings) != 1 {
t.Fatalf("findings = %d, want 1 (unfiltered: the gate never ran)", len(tc.prReview.findings))
}
}
func TestStagePRPrecisionGateDropsFindingsMarkedNotWorthPosting(t *testing.T) {
tc := baseTaskContext(t)
tc.Cfg.Review.PrecisionGate = true
tc.prReview = &prReviewState{
snapshotDir: t.TempDir(),
findings: []prreview.Finding{
{Title: "concrete", Severity: prreview.SeverityImportant},
{Title: "nitpick", Severity: prreview.SeverityNitpick},
},
}
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
if stage == "precision-gate" {
return captureResult("precision_gate", map[string]any{
"verdicts": []map[string]any{{"index": 0, "keep": true}, {"index": 1, "keep": false}},
}), nil
}
return passedResult("ok"), nil
}}
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
t.Fatalf("precision-gate: %v", err)
}
if len(tc.prReview.findings) != 1 || tc.prReview.findings[0].Title != "concrete" {
t.Fatalf("findings = %+v, want only the concrete finding kept", tc.prReview.findings)
}
}
func TestStagePRPrecisionGateFailedCallKeepsEveryFinding(t *testing.T) {
tc := baseTaskContext(t)
tc.Cfg.Review.PrecisionGate = true
tc.prReview = &prReviewState{
snapshotDir: t.TempDir(),
findings: []prreview.Finding{
{Title: "f0", Severity: prreview.SeverityImportant},
{Title: "f1", Severity: prreview.SeverityNitpick},
},
}
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
return agentexec.Result{Status: agentexec.StatusParked}, nil
}}
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
t.Fatalf("precision-gate: %v", err)
}
if len(tc.prReview.findings) != 2 {
t.Fatalf("findings = %d, want 2 (recall-first: a failed call filters nothing)", len(tc.prReview.findings))
}
}
func TestStagePRPrecisionGateRunsNoAgentCallWhenNoFindings(t *testing.T) {
tc := baseTaskContext(t)
tc.Cfg.Review.PrecisionGate = true
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
t.Fatal("no agent call expected with zero findings")
return agentexec.Result{}, nil
}}
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
t.Fatalf("precision-gate: %v", err)
}
}
func TestStagePROperatorApprovalWaitsWhenConfiguredAndFindingsExist(t *testing.T) {
tc := baseTaskContext(t)
tc.Cfg.Review.ApproveBeforePost = true
tc.prReview = &prReviewState{
scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}},
}
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
t.Fatalf("operator-approval: %v", err)
}
if tc.Outcome.Status != StatusWaitingHuman {
t.Fatalf("outcome status = %q, want %q", tc.Outcome.Status, StatusWaitingHuman)
}
}
func TestStagePROperatorApprovalSkipsWhenConfigOff(t *testing.T) {
tc := baseTaskContext(t)
tc.prReview = &prReviewState{
scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}},
}
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
t.Fatalf("operator-approval: %v", err)
}
if tc.Outcome.Status != "" {
t.Fatalf("outcome status = %q, want empty (config off, never waits)", tc.Outcome.Status)
}
}
func TestStagePROperatorApprovalNeverWaitsWithNoFindings(t *testing.T) {
tc := baseTaskContext(t)
tc.Cfg.Review.ApproveBeforePost = true
tc.prReview = &prReviewState{}
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
t.Fatalf("operator-approval: %v", err)
}
if tc.Outcome.Status != "" {
t.Fatalf("outcome status = %q, want empty (zero findings never wait, per the PRD)", tc.Outcome.Status)
}
}
@@ -44,6 +44,127 @@ func stagePRVerification() Stage {
}}
}
// stagePRPrecisionGate is the PRD's Precision dial: an optional
// post-worthiness pass, active only when review.precision_gate is set,
// between phase 4 (review) and phase 5 (verification). It keeps every
// concrete, evidenced defect and drops nitpicks, style and unverifiable
// claims, keeping a finding when unsure -- trading recall for precision and
// cutting verification cost by filtering early. It is off by default: the
// pipeline is recall-first.
func stagePRPrecisionGate() Stage {
return Stage{Name: "precision-gate", Run: func(ctx context.Context, tc *TaskContext) error {
if !tc.Cfg.Review.PrecisionGate || len(tc.prReview.findings) == 0 {
return nil
}
if budgetExhausted(tc, "precision-gate") {
skipPhase(tc, "precision-gate")
return nil
}
tc.prReview.findings = runPrecisionGate(ctx, tc, tc.prReview.findings)
return nil
}}
}
// precisionGateSchema is the precision gate's capture-tool schema: one
// keep/drop verdict per finding index, the same index-referencing shape the
// evidence verifier and adversary already use.
var precisionGateSchema = json.RawMessage(`{
"type": "object",
"properties": {
"verdicts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"index": {"type": "integer"},
"keep": {"type": "boolean"}
},
"required": ["index", "keep"]
}
}
},
"required": ["verdicts"]
}`)
// runPrecisionGate runs the precision gate's single classification-role call
// over every current finding. A failed or malformed call keeps every finding
// unfiltered -- the same recall-first fallback runEvidenceVerifier and
// runAdversary already use: a gate that could not run is not evidence any
// finding should drop.
func runPrecisionGate(ctx context.Context, tc *TaskContext, findings []prreview.Finding) []prreview.Finding {
var listing strings.Builder
for i, f := range findings {
fmt.Fprintf(&listing, "Finding %d [%s]: %s (%s:%d)\n%s\nEvidence: %s\n\n",
i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence)
}
mission := fmt.Sprintf(
"Decide which of these findings are worth posting to a human reviewer. Keep every "+
"concrete, evidenced defect. Drop nitpicks, pure style complaints, and any claim "+
"you cannot verify from the evidence shown. Keep a finding when you are unsure.\n\n"+
"%s"+
"Call precision_gate exactly once with one verdict per finding index, then call "+
"finish with status \"passed\".",
listing.String(),
)
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "precision-gate", "classification", mission, 15, []agentexec.CaptureTool{{
Name: "precision_gate", Description: "Record each finding's keep/drop verdict. Call exactly once, before finish.",
Parameters: precisionGateSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
}})
if err != nil || res.Status != agentexec.StatusPassed {
return findings //nolint:nilerr // recall-first: a gate that could not run is not evidence any finding should drop
}
calls := res.Captures["precision_gate"]
if len(calls) != 1 {
return findings
}
var captured struct {
Verdicts []struct {
Index int `json:"index"`
Keep bool `json:"keep"`
} `json:"verdicts"`
}
if json.Unmarshal(calls[0], &captured) != nil {
return findings
}
drop := make(map[int]bool, len(captured.Verdicts))
for _, v := range captured.Verdicts {
if !v.Keep {
drop[v.Index] = true
}
}
kept := make([]prreview.Finding, 0, len(findings))
for i, f := range findings {
if drop[i] {
continue
}
kept = append(kept, f)
}
return kept
}
// stagePROperatorApproval is the PRD's Operator approval gate: active only
// when review.approve_before_post is set, it runs immediately after
// synthesis and before the merge gate, so an operator's re-review (which
// reruns phases 3 to 8 with instructions folded in) never has to undo a
// merge-gate verdict computed on a since-changed finding set. A review with
// no findings never waits, per the PRD, and continues straight to the merge
// gate. Setting tc.Outcome here ends the workflow run at this stage --
// docs/prds/execution-tree-state-machine.md's engine loop stops as soon as a
// stage sets a non-empty Outcome.Status -- so the merge gate and output
// stages never run until an operator's response requeues the task.
func stagePROperatorApproval() Stage {
return Stage{Name: "operator-approval", Run: func(_ context.Context, tc *TaskContext) error {
if !tc.Cfg.Review.ApproveBeforePost || len(tc.prReview.scored) == 0 {
return nil
}
tc.Outcome = Outcome{
Status: StatusWaitingHuman,
Detail: fmt.Sprintf("%d finding(s) awaiting operator review before posting", len(tc.prReview.scored)),
}
return nil
}}
}
// evidencePackageFor extracts a finding's evidence package from the
// snapshot, reusing afbk.1's ExtractEvidence. A file the snapshot no longer
// has (a finding pointing at a path that does not exist) extracts an empty