feat(workflow): pr-review precision dial and operator-approval gate
Add review.precision_gate and review.approve_before_post to config, threaded through TaskConfig/ForTask/ToConfig so the pr-review workflow reads them under archie-agent too, not just in-process. stagePRPrecisionGate runs between review and verification, filtering findings down to concrete, evidenced defects when the dial is on; stagePROperatorApproval ends the run in waiting_human right after synthesis, before the merge gate, when approval is required and findings exist. Both are off by default and fail open on a broken agent call, matching the pipeline's existing recall-first fallbacks. Filed follow-ups: afbk.11 for the actual operator-response resume mechanics (post/reject/re-review with instructions, the 2-round cap -- genuinely needs either a small Task schema addition or a documented Notes-encoding decision, a call for whoever wires afbk.7's triggers) and afbk.12 for exposing the two flags in the dashboard Config tab.
This commit is contained in:
8 files changed
+291
-7
No files matched your search
@@ -117,6 +117,7 @@ var reloadableFields = map[string]bool{
|
||||
"Models": true,
|
||||
"ModelLimits": true,
|
||||
"Notify": true,
|
||||
"Review": true,
|
||||
// BotUser/BotEmail used to be poll-path only; ForTask now carries both
|
||||
// into the task snapshot as well, because the sandboxed worker builds
|
||||
// its own worktree.Manager and signs its own commits (agentworker
|
||||
|
||||
@@ -210,6 +210,23 @@ type Forge struct {
|
||||
WebhookAddr string `toml:"webhook_addr" yaml:"webhook_addr"`
|
||||
}
|
||||
|
||||
// Review configures the pr-review workflow (docs/prds/pr-review-agent.md).
|
||||
// Both fields are off by default: the pipeline is recall-first, and posts
|
||||
// automatically, unless an operator opts into either dial.
|
||||
type Review struct {
|
||||
// PrecisionGate runs a post-worthiness pass after phase 4 (review) and
|
||||
// before phase 5 (verification): it keeps every concrete, evidenced
|
||||
// defect, drops nitpicks, style and unverifiable claims, and keeps a
|
||||
// finding when unsure. It trades recall for precision and cuts
|
||||
// verification cost by filtering early.
|
||||
PrecisionGate bool `toml:"precision_gate" yaml:"precision_gate"`
|
||||
// ApproveBeforePost ends the pipeline in waiting_human right after
|
||||
// synthesis, before the merge gate runs, so an operator can choose which
|
||||
// findings to post, reject the review, or ask for a re-review with
|
||||
// instructions. A review with no findings never waits.
|
||||
ApproveBeforePost bool `toml:"approve_before_post" yaml:"approve_before_post"`
|
||||
}
|
||||
|
||||
// Dispatch configures how archied discovers work and reflects task state
|
||||
// onto the forge via labels and reactions.
|
||||
type Dispatch struct {
|
||||
@@ -486,6 +503,7 @@ type Config struct {
|
||||
|
||||
Forge Forge `toml:"forge" yaml:"forge"`
|
||||
Dispatch Dispatch `toml:"dispatch" yaml:"dispatch"`
|
||||
Review Review `toml:"review" yaml:"review"`
|
||||
|
||||
// Models maps a role ("planner", "builder", "triage") to a runtime
|
||||
// model ref ("provider/model").
|
||||
@@ -604,6 +622,7 @@ type TaskConfig struct {
|
||||
Notify Notify `json:"notify"`
|
||||
Forge TaskForge `json:"forge"`
|
||||
ToolPolicy ToolPolicy `json:"tool_policy"`
|
||||
Review Review `json:"review"`
|
||||
}
|
||||
|
||||
// DiffCapOf returns a DiffCapLines value for n. It exists because the field is
|
||||
@@ -644,6 +663,7 @@ func (c Config) ForTask() TaskConfig {
|
||||
Notify: c.Notify,
|
||||
Forge: TaskForge{Host: c.Forge.Host},
|
||||
ToolPolicy: c.Tools.Policy,
|
||||
Review: c.Review,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -663,6 +683,7 @@ func (tc TaskConfig) ToConfig() Config {
|
||||
Notify: tc.Notify,
|
||||
Forge: Forge{Host: tc.Forge.Host},
|
||||
Tools: ToolsConfig{Policy: tc.ToolPolicy},
|
||||
Review: tc.Review,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -157,8 +157,9 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
|
||||
Providers: map[string]Provider{
|
||||
"secret": {APIKeyEnv: "TOP_SECRET_PROVIDER_TOKEN"},
|
||||
},
|
||||
NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"},
|
||||
Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}},
|
||||
NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"},
|
||||
Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}},
|
||||
Review: Review{PrecisionGate: true, ApproveBeforePost: true},
|
||||
}
|
||||
|
||||
want := TaskConfig{
|
||||
@@ -171,6 +172,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
|
||||
Notify: cfg.Notify,
|
||||
Forge: TaskForge{Host: cfg.Forge.Host},
|
||||
ToolPolicy: cfg.Tools.Policy,
|
||||
Review: cfg.Review,
|
||||
}
|
||||
got := cfg.ForTask()
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
@@ -197,7 +199,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy")
|
||||
assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy", "review")
|
||||
|
||||
var forgePayload map[string]json.RawMessage
|
||||
var payload map[string]json.RawMessage
|
||||
|
||||
@@ -2098,7 +2098,7 @@ func TestDispatchCapturesTheAttemptsEffectiveConfig(t *testing.T) {
|
||||
// repositories, identities or lock state.
|
||||
wantKeys := []string{
|
||||
"bot_email", "bot_user", "budgets", "diff_cap_lines", "dispatch",
|
||||
"forge", "model_limits", "models", "notify", "tool_policy",
|
||||
"forge", "model_limits", "models", "notify", "review", "tool_policy",
|
||||
}
|
||||
gotKeys := make([]string, 0, len(doc))
|
||||
for key := range doc {
|
||||
|
||||
@@ -42,6 +42,7 @@ var outcomeSites = []outcomeSite{
|
||||
{"feasibility.go", "StatusClosedWontDo", "feasibility decided this work should not be done"},
|
||||
{"feasibility.go", "StatusWaitingHuman", "the PRD is delivered and the run waits for a go/no-go"},
|
||||
{"prreview_stages.go", "StatusCompleted", "the pr-review pipeline posted its findings (or found none to post)"},
|
||||
{"prreview_stages_verify.go", "StatusWaitingHuman", "review.approve_before_post is set and synthesis produced findings: the run waits for an operator decision before the merge gate"},
|
||||
{"remediate.go", "StatusParked", "the remediation round cap is spent"},
|
||||
{"remediate.go", "StatusPROpen", "a remediation round pushed the task back to its open PR"},
|
||||
{"review.go", "StatusParked", "review is enabled but no reviewer is wired"},
|
||||
|
||||
@@ -106,7 +106,7 @@ var prReviewTotalBudget = prreview.Budget{MaxTokens: 2_000_000, WallClock: 30 *
|
||||
// Intake is not tracked: it must always run, since the depth, the AI-score
|
||||
// and the budget's own runStart all come from it, and every later phase and
|
||||
// this list itself depend on its output existing.
|
||||
var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "verification", "coverage-consistency", "merge-gate", "output"}
|
||||
var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "precision-gate", "verification", "coverage-consistency", "merge-gate", "output"}
|
||||
|
||||
// budgetExhausted reports whether phase has already spent its cumulative
|
||||
// share of the run's total budget, checked once before that phase's
|
||||
@@ -142,8 +142,11 @@ const prReviewConcurrency = 8
|
||||
// PRReview is the pull request reviewer: a fixed pipeline, not a single
|
||||
// agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2,
|
||||
// 3, 4, 7 and 9; archie-core-afbk.4 added 5 and 6 (verification,
|
||||
// coverage/consistency); archie-core-afbk.5 adds phase 8 (the merge gate)
|
||||
// and the budget shares/skipped-phase reporting that run throughout.
|
||||
// coverage/consistency); archie-core-afbk.5 added phase 8 (the merge gate)
|
||||
// and the budget shares/skipped-phase reporting that run throughout;
|
||||
// archie-core-afbk.8 adds the precision dial (between review and
|
||||
// verification) and the operator-approval gate (between synthesis and the
|
||||
// merge gate), both off by default and each gated by its own config flag.
|
||||
func PRReview() Workflow {
|
||||
return Workflow{
|
||||
Name: "pr-review",
|
||||
@@ -152,9 +155,11 @@ func PRReview() Workflow {
|
||||
stagePRAnatomy(),
|
||||
stagePRLenses(),
|
||||
stagePRReview(),
|
||||
stagePRPrecisionGate(),
|
||||
stagePRVerification(),
|
||||
stagePRCoverageConsistency(),
|
||||
stagePRSynthesis(),
|
||||
stagePROperatorApproval(),
|
||||
stagePRMergeGate(),
|
||||
stagePROutput(),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/prreview"
|
||||
)
|
||||
|
||||
func TestStagePRPrecisionGateSkippedWhenConfigOff(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
findings: []prreview.Finding{{Title: "f0", Severity: prreview.SeverityNitpick}},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call expected: review.precision_gate is off")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 {
|
||||
t.Fatalf("findings = %d, want 1 (unfiltered: the gate never ran)", len(tc.prReview.findings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRPrecisionGateDropsFindingsMarkedNotWorthPosting(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.PrecisionGate = true
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
findings: []prreview.Finding{
|
||||
{Title: "concrete", Severity: prreview.SeverityImportant},
|
||||
{Title: "nitpick", Severity: prreview.SeverityNitpick},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) {
|
||||
if stage == "precision-gate" {
|
||||
return captureResult("precision_gate", map[string]any{
|
||||
"verdicts": []map[string]any{{"index": 0, "keep": true}, {"index": 1, "keep": false}},
|
||||
}), nil
|
||||
}
|
||||
return passedResult("ok"), nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 1 || tc.prReview.findings[0].Title != "concrete" {
|
||||
t.Fatalf("findings = %+v, want only the concrete finding kept", tc.prReview.findings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRPrecisionGateFailedCallKeepsEveryFinding(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.PrecisionGate = true
|
||||
tc.prReview = &prReviewState{
|
||||
snapshotDir: t.TempDir(),
|
||||
findings: []prreview.Finding{
|
||||
{Title: "f0", Severity: prreview.SeverityImportant},
|
||||
{Title: "f1", Severity: prreview.SeverityNitpick},
|
||||
},
|
||||
}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
return agentexec.Result{Status: agentexec.StatusParked}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
if len(tc.prReview.findings) != 2 {
|
||||
t.Fatalf("findings = %d, want 2 (recall-first: a failed call filters nothing)", len(tc.prReview.findings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePRPrecisionGateRunsNoAgentCallWhenNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.PrecisionGate = true
|
||||
tc.prReview = &prReviewState{snapshotDir: t.TempDir()}
|
||||
tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) {
|
||||
t.Fatal("no agent call expected with zero findings")
|
||||
return agentexec.Result{}, nil
|
||||
}}
|
||||
|
||||
if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("precision-gate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROperatorApprovalWaitsWhenConfiguredAndFindingsExist(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.ApproveBeforePost = true
|
||||
tc.prReview = &prReviewState{
|
||||
scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}},
|
||||
}
|
||||
|
||||
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("operator-approval: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != StatusWaitingHuman {
|
||||
t.Fatalf("outcome status = %q, want %q", tc.Outcome.Status, StatusWaitingHuman)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROperatorApprovalSkipsWhenConfigOff(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.prReview = &prReviewState{
|
||||
scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}},
|
||||
}
|
||||
|
||||
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("operator-approval: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
t.Fatalf("outcome status = %q, want empty (config off, never waits)", tc.Outcome.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStagePROperatorApprovalNeverWaitsWithNoFindings(t *testing.T) {
|
||||
tc := baseTaskContext(t)
|
||||
tc.Cfg.Review.ApproveBeforePost = true
|
||||
tc.prReview = &prReviewState{}
|
||||
|
||||
if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("operator-approval: %v", err)
|
||||
}
|
||||
if tc.Outcome.Status != "" {
|
||||
t.Fatalf("outcome status = %q, want empty (zero findings never wait, per the PRD)", tc.Outcome.Status)
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,127 @@ func stagePRVerification() Stage {
|
||||
}}
|
||||
}
|
||||
|
||||
// stagePRPrecisionGate is the PRD's Precision dial: an optional
|
||||
// post-worthiness pass, active only when review.precision_gate is set,
|
||||
// between phase 4 (review) and phase 5 (verification). It keeps every
|
||||
// concrete, evidenced defect and drops nitpicks, style and unverifiable
|
||||
// claims, keeping a finding when unsure -- trading recall for precision and
|
||||
// cutting verification cost by filtering early. It is off by default: the
|
||||
// pipeline is recall-first.
|
||||
func stagePRPrecisionGate() Stage {
|
||||
return Stage{Name: "precision-gate", Run: func(ctx context.Context, tc *TaskContext) error {
|
||||
if !tc.Cfg.Review.PrecisionGate || len(tc.prReview.findings) == 0 {
|
||||
return nil
|
||||
}
|
||||
if budgetExhausted(tc, "precision-gate") {
|
||||
skipPhase(tc, "precision-gate")
|
||||
return nil
|
||||
}
|
||||
tc.prReview.findings = runPrecisionGate(ctx, tc, tc.prReview.findings)
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// precisionGateSchema is the precision gate's capture-tool schema: one
|
||||
// keep/drop verdict per finding index, the same index-referencing shape the
|
||||
// evidence verifier and adversary already use.
|
||||
var precisionGateSchema = json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"verdicts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {"type": "integer"},
|
||||
"keep": {"type": "boolean"}
|
||||
},
|
||||
"required": ["index", "keep"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["verdicts"]
|
||||
}`)
|
||||
|
||||
// runPrecisionGate runs the precision gate's single classification-role call
|
||||
// over every current finding. A failed or malformed call keeps every finding
|
||||
// unfiltered -- the same recall-first fallback runEvidenceVerifier and
|
||||
// runAdversary already use: a gate that could not run is not evidence any
|
||||
// finding should drop.
|
||||
func runPrecisionGate(ctx context.Context, tc *TaskContext, findings []prreview.Finding) []prreview.Finding {
|
||||
var listing strings.Builder
|
||||
for i, f := range findings {
|
||||
fmt.Fprintf(&listing, "Finding %d [%s]: %s (%s:%d)\n%s\nEvidence: %s\n\n",
|
||||
i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence)
|
||||
}
|
||||
mission := fmt.Sprintf(
|
||||
"Decide which of these findings are worth posting to a human reviewer. Keep every "+
|
||||
"concrete, evidenced defect. Drop nitpicks, pure style complaints, and any claim "+
|
||||
"you cannot verify from the evidence shown. Keep a finding when you are unsure.\n\n"+
|
||||
"%s"+
|
||||
"Call precision_gate exactly once with one verdict per finding index, then call "+
|
||||
"finish with status \"passed\".",
|
||||
listing.String(),
|
||||
)
|
||||
res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "precision-gate", "classification", mission, 15, []agentexec.CaptureTool{{
|
||||
Name: "precision_gate", Description: "Record each finding's keep/drop verdict. Call exactly once, before finish.",
|
||||
Parameters: precisionGateSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1,
|
||||
}})
|
||||
if err != nil || res.Status != agentexec.StatusPassed {
|
||||
return findings //nolint:nilerr // recall-first: a gate that could not run is not evidence any finding should drop
|
||||
}
|
||||
calls := res.Captures["precision_gate"]
|
||||
if len(calls) != 1 {
|
||||
return findings
|
||||
}
|
||||
var captured struct {
|
||||
Verdicts []struct {
|
||||
Index int `json:"index"`
|
||||
Keep bool `json:"keep"`
|
||||
} `json:"verdicts"`
|
||||
}
|
||||
if json.Unmarshal(calls[0], &captured) != nil {
|
||||
return findings
|
||||
}
|
||||
drop := make(map[int]bool, len(captured.Verdicts))
|
||||
for _, v := range captured.Verdicts {
|
||||
if !v.Keep {
|
||||
drop[v.Index] = true
|
||||
}
|
||||
}
|
||||
kept := make([]prreview.Finding, 0, len(findings))
|
||||
for i, f := range findings {
|
||||
if drop[i] {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, f)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// stagePROperatorApproval is the PRD's Operator approval gate: active only
|
||||
// when review.approve_before_post is set, it runs immediately after
|
||||
// synthesis and before the merge gate, so an operator's re-review (which
|
||||
// reruns phases 3 to 8 with instructions folded in) never has to undo a
|
||||
// merge-gate verdict computed on a since-changed finding set. A review with
|
||||
// no findings never waits, per the PRD, and continues straight to the merge
|
||||
// gate. Setting tc.Outcome here ends the workflow run at this stage --
|
||||
// docs/prds/execution-tree-state-machine.md's engine loop stops as soon as a
|
||||
// stage sets a non-empty Outcome.Status -- so the merge gate and output
|
||||
// stages never run until an operator's response requeues the task.
|
||||
func stagePROperatorApproval() Stage {
|
||||
return Stage{Name: "operator-approval", Run: func(_ context.Context, tc *TaskContext) error {
|
||||
if !tc.Cfg.Review.ApproveBeforePost || len(tc.prReview.scored) == 0 {
|
||||
return nil
|
||||
}
|
||||
tc.Outcome = Outcome{
|
||||
Status: StatusWaitingHuman,
|
||||
Detail: fmt.Sprintf("%d finding(s) awaiting operator review before posting", len(tc.prReview.scored)),
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
}
|
||||
|
||||
// evidencePackageFor extracts a finding's evidence package from the
|
||||
// snapshot, reusing afbk.1's ExtractEvidence. A file the snapshot no longer
|
||||
// has (a finding pointing at a path that does not exist) extracts an empty
|
||||
|
||||
Reference in new issue
Block a user