diff --git a/internal/app/archied/reload.go b/internal/app/archied/reload.go index 5dc1649f8..a194b5015 100644 --- a/internal/app/archied/reload.go +++ b/internal/app/archied/reload.go @@ -117,6 +117,7 @@ var reloadableFields = map[string]bool{ "Models": true, "ModelLimits": true, "Notify": true, + "Review": true, // BotUser/BotEmail used to be poll-path only; ForTask now carries both // into the task snapshot as well, because the sandboxed worker builds // its own worktree.Manager and signs its own commits (agentworker diff --git a/internal/config/config.go b/internal/config/config.go index faff0caf7..98d0a5a80 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -210,6 +210,23 @@ type Forge struct { WebhookAddr string `toml:"webhook_addr" yaml:"webhook_addr"` } +// Review configures the pr-review workflow (docs/prds/pr-review-agent.md). +// Both fields are off by default: the pipeline is recall-first, and posts +// automatically, unless an operator opts into either dial. +type Review struct { + // PrecisionGate runs a post-worthiness pass after phase 4 (review) and + // before phase 5 (verification): it keeps every concrete, evidenced + // defect, drops nitpicks, style and unverifiable claims, and keeps a + // finding when unsure. It trades recall for precision and cuts + // verification cost by filtering early. + PrecisionGate bool `toml:"precision_gate" yaml:"precision_gate"` + // ApproveBeforePost ends the pipeline in waiting_human right after + // synthesis, before the merge gate runs, so an operator can choose which + // findings to post, reject the review, or ask for a re-review with + // instructions. A review with no findings never waits. + ApproveBeforePost bool `toml:"approve_before_post" yaml:"approve_before_post"` +} + // Dispatch configures how archied discovers work and reflects task state // onto the forge via labels and reactions. type Dispatch struct { @@ -486,6 +503,7 @@ type Config struct { Forge Forge `toml:"forge" yaml:"forge"` Dispatch Dispatch `toml:"dispatch" yaml:"dispatch"` + Review Review `toml:"review" yaml:"review"` // Models maps a role ("planner", "builder", "triage") to a runtime // model ref ("provider/model"). @@ -604,6 +622,7 @@ type TaskConfig struct { Notify Notify `json:"notify"` Forge TaskForge `json:"forge"` ToolPolicy ToolPolicy `json:"tool_policy"` + Review Review `json:"review"` } // DiffCapOf returns a DiffCapLines value for n. It exists because the field is @@ -644,6 +663,7 @@ func (c Config) ForTask() TaskConfig { Notify: c.Notify, Forge: TaskForge{Host: c.Forge.Host}, ToolPolicy: c.Tools.Policy, + Review: c.Review, } } @@ -663,6 +683,7 @@ func (tc TaskConfig) ToConfig() Config { Notify: tc.Notify, Forge: Forge{Host: tc.Forge.Host}, Tools: ToolsConfig{Policy: tc.ToolPolicy}, + Review: tc.Review, } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3f8cca567..172e43e82 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -157,8 +157,9 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) { Providers: map[string]Provider{ "secret": {APIKeyEnv: "TOP_SECRET_PROVIDER_TOKEN"}, }, - NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"}, - Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}}, + NATS: NATSConfig{TokenEnv: "TOP_SECRET_NATS_TOKEN"}, + Tools: ToolsConfig{Policy: ToolPolicy{MaxResultChars: 50_000, SpillDir: "/var/tmp/archie-spill"}}, + Review: Review{PrecisionGate: true, ApproveBeforePost: true}, } want := TaskConfig{ @@ -171,6 +172,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) { Notify: cfg.Notify, Forge: TaskForge{Host: cfg.Forge.Host}, ToolPolicy: cfg.Tools.Policy, + Review: cfg.Review, } got := cfg.ForTask() if !reflect.DeepEqual(got, want) { @@ -197,7 +199,7 @@ func TestConfigForTaskJSONRoundTrip(t *testing.T) { if err != nil { t.Fatal(err) } - assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy") + assertJSONKeys(t, data, "bot_user", "bot_email", "models", "budgets", "dispatch", "diff_cap_lines", "notify", "forge", "tool_policy", "review") var forgePayload map[string]json.RawMessage var payload map[string]json.RawMessage diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index bc5ceb227..7c81b066d 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -2098,7 +2098,7 @@ func TestDispatchCapturesTheAttemptsEffectiveConfig(t *testing.T) { // repositories, identities or lock state. wantKeys := []string{ "bot_email", "bot_user", "budgets", "diff_cap_lines", "dispatch", - "forge", "model_limits", "models", "notify", "tool_policy", + "forge", "model_limits", "models", "notify", "review", "tool_policy", } gotKeys := make([]string, 0, len(doc)) for key := range doc { diff --git a/internal/domain/workflow/outcome_sites_test.go b/internal/domain/workflow/outcome_sites_test.go index 3fe5bc5a3..9ba47e003 100644 --- a/internal/domain/workflow/outcome_sites_test.go +++ b/internal/domain/workflow/outcome_sites_test.go @@ -42,6 +42,7 @@ var outcomeSites = []outcomeSite{ {"feasibility.go", "StatusClosedWontDo", "feasibility decided this work should not be done"}, {"feasibility.go", "StatusWaitingHuman", "the PRD is delivered and the run waits for a go/no-go"}, {"prreview_stages.go", "StatusCompleted", "the pr-review pipeline posted its findings (or found none to post)"}, + {"prreview_stages_verify.go", "StatusWaitingHuman", "review.approve_before_post is set and synthesis produced findings: the run waits for an operator decision before the merge gate"}, {"remediate.go", "StatusParked", "the remediation round cap is spent"}, {"remediate.go", "StatusPROpen", "a remediation round pushed the task back to its open PR"}, {"review.go", "StatusParked", "review is enabled but no reviewer is wired"}, diff --git a/internal/domain/workflow/prreview_stages.go b/internal/domain/workflow/prreview_stages.go index 1761f46bc..b7569d7e1 100644 --- a/internal/domain/workflow/prreview_stages.go +++ b/internal/domain/workflow/prreview_stages.go @@ -106,7 +106,7 @@ var prReviewTotalBudget = prreview.Budget{MaxTokens: 2_000_000, WallClock: 30 * // Intake is not tracked: it must always run, since the depth, the AI-score // and the budget's own runStart all come from it, and every later phase and // this list itself depend on its output existing. -var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "verification", "coverage-consistency", "merge-gate", "output"} +var prReviewBudgetPhases = []string{"anatomy", "lenses", "review", "precision-gate", "verification", "coverage-consistency", "merge-gate", "output"} // budgetExhausted reports whether phase has already spent its cumulative // share of the run's total budget, checked once before that phase's @@ -142,8 +142,11 @@ const prReviewConcurrency = 8 // PRReview is the pull request reviewer: a fixed pipeline, not a single // agent (docs/prds/pr-review-agent.md). archie-core-afbk.3 wired phases 1, 2, // 3, 4, 7 and 9; archie-core-afbk.4 added 5 and 6 (verification, -// coverage/consistency); archie-core-afbk.5 adds phase 8 (the merge gate) -// and the budget shares/skipped-phase reporting that run throughout. +// coverage/consistency); archie-core-afbk.5 added phase 8 (the merge gate) +// and the budget shares/skipped-phase reporting that run throughout; +// archie-core-afbk.8 adds the precision dial (between review and +// verification) and the operator-approval gate (between synthesis and the +// merge gate), both off by default and each gated by its own config flag. func PRReview() Workflow { return Workflow{ Name: "pr-review", @@ -152,9 +155,11 @@ func PRReview() Workflow { stagePRAnatomy(), stagePRLenses(), stagePRReview(), + stagePRPrecisionGate(), stagePRVerification(), stagePRCoverageConsistency(), stagePRSynthesis(), + stagePROperatorApproval(), stagePRMergeGate(), stagePROutput(), }, diff --git a/internal/domain/workflow/prreview_stages_gate_test.go b/internal/domain/workflow/prreview_stages_gate_test.go new file mode 100644 index 000000000..d9a0cb0ce --- /dev/null +++ b/internal/domain/workflow/prreview_stages_gate_test.go @@ -0,0 +1,133 @@ +package workflow + +import ( + "context" + "testing" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/domain/workflow/prreview" +) + +func TestStagePRPrecisionGateSkippedWhenConfigOff(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{ + snapshotDir: t.TempDir(), + findings: []prreview.Finding{{Title: "f0", Severity: prreview.SeverityNitpick}}, + } + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + t.Fatal("no agent call expected: review.precision_gate is off") + return agentexec.Result{}, nil + }} + + if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil { + t.Fatalf("precision-gate: %v", err) + } + if len(tc.prReview.findings) != 1 { + t.Fatalf("findings = %d, want 1 (unfiltered: the gate never ran)", len(tc.prReview.findings)) + } +} + +func TestStagePRPrecisionGateDropsFindingsMarkedNotWorthPosting(t *testing.T) { + tc := baseTaskContext(t) + tc.Cfg.Review.PrecisionGate = true + tc.prReview = &prReviewState{ + snapshotDir: t.TempDir(), + findings: []prreview.Finding{ + {Title: "concrete", Severity: prreview.SeverityImportant}, + {Title: "nitpick", Severity: prreview.SeverityNitpick}, + }, + } + tc.Agent = &concurrentAgentRunner{byStage: func(stage string) (agentexec.Result, error) { + if stage == "precision-gate" { + return captureResult("precision_gate", map[string]any{ + "verdicts": []map[string]any{{"index": 0, "keep": true}, {"index": 1, "keep": false}}, + }), nil + } + return passedResult("ok"), nil + }} + + if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil { + t.Fatalf("precision-gate: %v", err) + } + if len(tc.prReview.findings) != 1 || tc.prReview.findings[0].Title != "concrete" { + t.Fatalf("findings = %+v, want only the concrete finding kept", tc.prReview.findings) + } +} + +func TestStagePRPrecisionGateFailedCallKeepsEveryFinding(t *testing.T) { + tc := baseTaskContext(t) + tc.Cfg.Review.PrecisionGate = true + tc.prReview = &prReviewState{ + snapshotDir: t.TempDir(), + findings: []prreview.Finding{ + {Title: "f0", Severity: prreview.SeverityImportant}, + {Title: "f1", Severity: prreview.SeverityNitpick}, + }, + } + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + return agentexec.Result{Status: agentexec.StatusParked}, nil + }} + + if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil { + t.Fatalf("precision-gate: %v", err) + } + if len(tc.prReview.findings) != 2 { + t.Fatalf("findings = %d, want 2 (recall-first: a failed call filters nothing)", len(tc.prReview.findings)) + } +} + +func TestStagePRPrecisionGateRunsNoAgentCallWhenNoFindings(t *testing.T) { + tc := baseTaskContext(t) + tc.Cfg.Review.PrecisionGate = true + tc.prReview = &prReviewState{snapshotDir: t.TempDir()} + tc.Agent = &concurrentAgentRunner{byStage: func(string) (agentexec.Result, error) { + t.Fatal("no agent call expected with zero findings") + return agentexec.Result{}, nil + }} + + if err := stagePRPrecisionGate().Run(context.Background(), tc); err != nil { + t.Fatalf("precision-gate: %v", err) + } +} + +func TestStagePROperatorApprovalWaitsWhenConfiguredAndFindingsExist(t *testing.T) { + tc := baseTaskContext(t) + tc.Cfg.Review.ApproveBeforePost = true + tc.prReview = &prReviewState{ + scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}}, + } + + if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil { + t.Fatalf("operator-approval: %v", err) + } + if tc.Outcome.Status != StatusWaitingHuman { + t.Fatalf("outcome status = %q, want %q", tc.Outcome.Status, StatusWaitingHuman) + } +} + +func TestStagePROperatorApprovalSkipsWhenConfigOff(t *testing.T) { + tc := baseTaskContext(t) + tc.prReview = &prReviewState{ + scored: []prreview.ScoredFinding{{Title: "f0", Score: 0.9}}, + } + + if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil { + t.Fatalf("operator-approval: %v", err) + } + if tc.Outcome.Status != "" { + t.Fatalf("outcome status = %q, want empty (config off, never waits)", tc.Outcome.Status) + } +} + +func TestStagePROperatorApprovalNeverWaitsWithNoFindings(t *testing.T) { + tc := baseTaskContext(t) + tc.Cfg.Review.ApproveBeforePost = true + tc.prReview = &prReviewState{} + + if err := stagePROperatorApproval().Run(context.Background(), tc); err != nil { + t.Fatalf("operator-approval: %v", err) + } + if tc.Outcome.Status != "" { + t.Fatalf("outcome status = %q, want empty (zero findings never wait, per the PRD)", tc.Outcome.Status) + } +} diff --git a/internal/domain/workflow/prreview_stages_verify.go b/internal/domain/workflow/prreview_stages_verify.go index 76d8ec3b1..0c3d2dcd2 100644 --- a/internal/domain/workflow/prreview_stages_verify.go +++ b/internal/domain/workflow/prreview_stages_verify.go @@ -44,6 +44,127 @@ func stagePRVerification() Stage { }} } +// stagePRPrecisionGate is the PRD's Precision dial: an optional +// post-worthiness pass, active only when review.precision_gate is set, +// between phase 4 (review) and phase 5 (verification). It keeps every +// concrete, evidenced defect and drops nitpicks, style and unverifiable +// claims, keeping a finding when unsure -- trading recall for precision and +// cutting verification cost by filtering early. It is off by default: the +// pipeline is recall-first. +func stagePRPrecisionGate() Stage { + return Stage{Name: "precision-gate", Run: func(ctx context.Context, tc *TaskContext) error { + if !tc.Cfg.Review.PrecisionGate || len(tc.prReview.findings) == 0 { + return nil + } + if budgetExhausted(tc, "precision-gate") { + skipPhase(tc, "precision-gate") + return nil + } + tc.prReview.findings = runPrecisionGate(ctx, tc, tc.prReview.findings) + return nil + }} +} + +// precisionGateSchema is the precision gate's capture-tool schema: one +// keep/drop verdict per finding index, the same index-referencing shape the +// evidence verifier and adversary already use. +var precisionGateSchema = json.RawMessage(`{ + "type": "object", + "properties": { + "verdicts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "index": {"type": "integer"}, + "keep": {"type": "boolean"} + }, + "required": ["index", "keep"] + } + } + }, + "required": ["verdicts"] +}`) + +// runPrecisionGate runs the precision gate's single classification-role call +// over every current finding. A failed or malformed call keeps every finding +// unfiltered -- the same recall-first fallback runEvidenceVerifier and +// runAdversary already use: a gate that could not run is not evidence any +// finding should drop. +func runPrecisionGate(ctx context.Context, tc *TaskContext, findings []prreview.Finding) []prreview.Finding { + var listing strings.Builder + for i, f := range findings { + fmt.Fprintf(&listing, "Finding %d [%s]: %s (%s:%d)\n%s\nEvidence: %s\n\n", + i, f.Severity, f.Title, f.File, f.LineStart, f.Body, f.Evidence) + } + mission := fmt.Sprintf( + "Decide which of these findings are worth posting to a human reviewer. Keep every "+ + "concrete, evidenced defect. Drop nitpicks, pure style complaints, and any claim "+ + "you cannot verify from the evidence shown. Keep a finding when you are unsure.\n\n"+ + "%s"+ + "Call precision_gate exactly once with one verdict per finding index, then call "+ + "finish with status \"passed\".", + listing.String(), + ) + res, err := runPRReviewAgentRecorded(ctx, tc, tc.prReview.snapshotDir, "precision-gate", "classification", mission, 15, []agentexec.CaptureTool{{ + Name: "precision_gate", Description: "Record each finding's keep/drop verdict. Call exactly once, before finish.", + Parameters: precisionGateSchema, RequiredFields: []string{"verdicts"}, MaxCalls: 1, + }}) + if err != nil || res.Status != agentexec.StatusPassed { + return findings //nolint:nilerr // recall-first: a gate that could not run is not evidence any finding should drop + } + calls := res.Captures["precision_gate"] + if len(calls) != 1 { + return findings + } + var captured struct { + Verdicts []struct { + Index int `json:"index"` + Keep bool `json:"keep"` + } `json:"verdicts"` + } + if json.Unmarshal(calls[0], &captured) != nil { + return findings + } + drop := make(map[int]bool, len(captured.Verdicts)) + for _, v := range captured.Verdicts { + if !v.Keep { + drop[v.Index] = true + } + } + kept := make([]prreview.Finding, 0, len(findings)) + for i, f := range findings { + if drop[i] { + continue + } + kept = append(kept, f) + } + return kept +} + +// stagePROperatorApproval is the PRD's Operator approval gate: active only +// when review.approve_before_post is set, it runs immediately after +// synthesis and before the merge gate, so an operator's re-review (which +// reruns phases 3 to 8 with instructions folded in) never has to undo a +// merge-gate verdict computed on a since-changed finding set. A review with +// no findings never waits, per the PRD, and continues straight to the merge +// gate. Setting tc.Outcome here ends the workflow run at this stage -- +// docs/prds/execution-tree-state-machine.md's engine loop stops as soon as a +// stage sets a non-empty Outcome.Status -- so the merge gate and output +// stages never run until an operator's response requeues the task. +func stagePROperatorApproval() Stage { + return Stage{Name: "operator-approval", Run: func(_ context.Context, tc *TaskContext) error { + if !tc.Cfg.Review.ApproveBeforePost || len(tc.prReview.scored) == 0 { + return nil + } + tc.Outcome = Outcome{ + Status: StatusWaitingHuman, + Detail: fmt.Sprintf("%d finding(s) awaiting operator review before posting", len(tc.prReview.scored)), + } + return nil + }} +} + // evidencePackageFor extracts a finding's evidence package from the // snapshot, reusing afbk.1's ExtractEvidence. A file the snapshot no longer // has (a finding pointing at a path that does not exist) extracts an empty